diff --git a/.github/CONTRIBUTING.md b/CONTRIBUTING.md
similarity index 100%
rename from .github/CONTRIBUTING.md
rename to CONTRIBUTING.md
diff --git a/README.textile b/README.textile
index 804f46a1811..be51570e271 100644
--- a/README.textile
+++ b/README.textile
@@ -147,7 +147,7 @@ curl -XPUT 'http://localhost:9200/kimchy/tweet/2' -d '
 }'
 </pre>
 
-The above will index information into the @kimchy@ index, with two types, @info@ and @tweet@. Each user will get his own special index.
+The above will index information into the @kimchy@ index, with two types, @info@ and @tweet@. Each user will get their own special index.
 
 Complete control on the index level is allowed. As an example, in the above case, we would want to change from the default 5 shards with 1 replica per index, to only 1 shard with 1 replica per index (== per twitter user). Here is how this can be done (the configuration can be in yaml as well):
 
diff --git a/Vagrantfile b/Vagrantfile
index 454d114f1a2..4f8ee7164f6 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -23,15 +23,15 @@
 
 Vagrant.configure(2) do |config|
   config.vm.define "ubuntu-1204" do |config|
-    config.vm.box = "ubuntu/precise64"
+    config.vm.box = "elastic/ubuntu-12.04-x86_64"
     ubuntu_common config
   end
   config.vm.define "ubuntu-1404" do |config|
-    config.vm.box = "ubuntu/trusty64"
+    config.vm.box = "elastic/ubuntu-14.04-x86_64"
     ubuntu_common config
   end
   config.vm.define "ubuntu-1504" do |config|
-    config.vm.box = "ubuntu/vivid64"
+    config.vm.box = "elastic/ubuntu-15.04-x86_64"
     ubuntu_common config, extra: <<-SHELL
       # Install Jayatana so we can work around it being present.
       [ -f /usr/share/java/jayatanaag.jar ] || install jayatana
@@ -41,44 +41,35 @@ Vagrant.configure(2) do |config|
   # get the sun jdk on there just aren't worth it. We have jessie for testing
   # debian and it works fine.
   config.vm.define "debian-8" do |config|
-    config.vm.box = "debian/jessie64"
-    deb_common config,
-      'echo deb http://http.debian.net/debian jessie-backports main > /etc/apt/sources.list.d/backports.list', 'backports'
+    config.vm.box = "elastic/debian-8-x86_64"
+    deb_common config, 'echo deb http://http.debian.net/debian jessie-backports main > /etc/apt/sources.list.d/backports.list', 'backports'
   end
   config.vm.define "centos-6" do |config|
-    config.vm.box = "boxcutter/centos67"
+    config.vm.box = "elastic/centos-6-x86_64"
     rpm_common config
   end
   config.vm.define "centos-7" do |config|
-    # There is a centos/7 box but it doesn't have rsync or virtualbox guest
-    # stuff on there so its slow to use. So chef it is....
-    config.vm.box = "boxcutter/centos71"
+    config.vm.box = "elastic/centos-7-x86_64"
+    rpm_common config
+  end
+  config.vm.define "oel-6" do |config|
+    config.vm.box = "elastic/oraclelinux-6-x86_64"
     rpm_common config
   end
-  # This box hangs _forever_ on ```yum check-update```. I have no idea why.
-  # config.vm.define "oel-6", autostart: false do |config|
-  #   config.vm.box = "boxcutter/oel66"
-  #   rpm_common(config)
-  # end
   config.vm.define "oel-7" do |config|
-    config.vm.box = "boxcutter/oel70"
+    config.vm.box = "elastic/oraclelinux-7-x86_64"
     rpm_common config
   end
   config.vm.define "fedora-22" do |config|
-    # Fedora hosts their own 'cloud' images that aren't in Vagrant's Atlas but
-    # and are missing required stuff like rsync. It'd be nice if we could use
-    # them but they much slower to get up and running then the boxcutter image.
-    config.vm.box = "boxcutter/fedora22"
+    config.vm.box = "elastic/fedora-22-x86_64"
     dnf_common config
   end
   config.vm.define "opensuse-13" do |config|
-    config.vm.box = "chef/opensuse-13"
-    config.vm.box_url = "http://opscode-vm-bento.s3.amazonaws.com/vagrant/virtualbox/opscode_opensuse-13.2-x86_64_chef-provisionerless.box"
+    config.vm.box = "elastic/opensuse-13-x86_64"
     opensuse_common config
   end
-  # The SLES boxes are not considered to be highest quality, but seem to be sufficient for a test run
   config.vm.define "sles-12" do |config|
-    config.vm.box = "idar/sles12"
+    config.vm.box = "elastic/sles-12-x86_64"
     sles_common config
   end
   # Switch the default share for the project root from /vagrant to
diff --git a/build.gradle b/build.gradle
index b419bf01e15..6ab00d73881 100644
--- a/build.gradle
+++ b/build.gradle
@@ -116,6 +116,7 @@ subprojects {
     "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar',
     "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:rpm',
     "org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:deb',
+    "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
   ]
   configurations.all {
     resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index ca78157bcf2..598be546f26 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -307,6 +307,12 @@ class BuildPlugin implements Plugin<Project> {
     /** Adds repositores used by ES dependencies */
     static void configureRepositories(Project project) {
         RepositoryHandler repos = project.repositories
+        if (System.getProperty("repos.mavenlocal") != null) {
+            // with -Drepos.mavenlocal=true we can force checking the local .m2 repo which is
+            // useful for development ie. bwc tests where we install stuff in the local repository
+            // such that we don't have to pass hardcoded files to gradle
+            repos.mavenLocal()
+        }
         repos.mavenCentral()
         repos.maven {
             name 'sonatype-snapshots'
@@ -407,6 +413,7 @@ class BuildPlugin implements Plugin<Project> {
             systemProperty 'jna.nosys', 'true'
             // default test sysprop values
             systemProperty 'tests.ifNoTests', 'fail'
+            // TODO: remove setting logging level via system property
             systemProperty 'es.logger.level', 'WARN'
             for (Map.Entry<String, String> property : System.properties.entrySet()) {
                 if (property.getKey().startsWith('tests.') ||
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy
index 7b949b3e1da..54699a52e76 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy
@@ -39,9 +39,6 @@ class PluginPropertiesExtension {
     @Input
     String classname
 
-    @Input
-    boolean isolated = true
-
     PluginPropertiesExtension(Project project) {
         name = project.name
         version = project.version
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy
index b5128817fb0..856c9255312 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy
@@ -54,12 +54,6 @@ class PluginPropertiesTask extends Copy {
             if (extension.classname == null) {
                 throw new InvalidUserDataException('classname is a required setting for esplugin')
             }
-            doFirst {
-                if (extension.isolated == false) {
-                    String warning = "WARNING: Disabling plugin isolation in ${project.path} is deprecated and will be removed in the future"
-                    logger.warn("${'=' * warning.length()}\n${warning}\n${'=' * warning.length()}")
-                }
-            }
             // configure property substitution
             from(templateFile)
             into(generatedResourcesDir)
@@ -80,7 +74,6 @@ class PluginPropertiesTask extends Copy {
             'version': stringSnap(extension.version),
             'elasticsearchVersion': stringSnap(VersionProperties.elasticsearch),
             'javaVersion': project.targetCompatibility as String,
-            'isolated': extension.isolated as String,
             'classname': extension.classname
         ]
     }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LoggerUsageTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LoggerUsageTask.groovy
new file mode 100644
index 00000000000..b280a74db58
--- /dev/null
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LoggerUsageTask.groovy
@@ -0,0 +1,98 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.precommit
+
+import org.elasticsearch.gradle.LoggedExec
+import org.gradle.api.file.FileCollection
+import org.gradle.api.tasks.InputFiles
+import org.gradle.api.tasks.OutputFile
+
+/**
+ * Runs LoggerUsageCheck on a set of directories.
+ */
+public class LoggerUsageTask extends LoggedExec {
+
+    /**
+     * We use a simple "marker" file that we touch when the task succeeds
+     * as the task output. This is compared against the modified time of the
+     * inputs (ie the jars/class files).
+     */
+    private File successMarker = new File(project.buildDir, 'markers/loggerUsage')
+
+    private FileCollection classpath;
+
+    private List<File> classDirectories;
+
+    public LoggerUsageTask() {
+        project.afterEvaluate {
+            dependsOn(classpath)
+            description = "Runs LoggerUsageCheck on ${classDirectories}"
+            executable = new File(project.javaHome, 'bin/java')
+            if (classDirectories == null) {
+                classDirectories = []
+                if (project.sourceSets.findByName("main") && project.sourceSets.main.output.classesDir.exists()) {
+                    classDirectories += [project.sourceSets.main.output.classesDir]
+                    dependsOn project.tasks.classes
+                }
+                if (project.sourceSets.findByName("test") && project.sourceSets.test.output.classesDir.exists()) {
+                    classDirectories += [project.sourceSets.test.output.classesDir]
+                    dependsOn project.tasks.testClasses
+                }
+            }
+            doFirst({
+                args('-cp', getClasspath().asPath, 'org.elasticsearch.test.loggerusage.ESLoggerUsageChecker')
+                getClassDirectories().each {
+                    args it.getAbsolutePath()
+                }
+            })
+            doLast({
+                successMarker.parentFile.mkdirs()
+                successMarker.setText("", 'UTF-8')
+            })
+        }
+    }
+
+    @InputFiles
+    FileCollection getClasspath() {
+        return classpath
+    }
+
+    void setClasspath(FileCollection classpath) {
+        this.classpath = classpath
+    }
+
+    @InputFiles
+    List<File> getClassDirectories() {
+        return classDirectories
+    }
+
+    void setClassDirectories(List<File> classDirectories) {
+        this.classDirectories = classDirectories
+    }
+
+    @OutputFile
+    File getSuccessMarker() {
+        return successMarker
+    }
+
+    void setSuccessMarker(File successMarker) {
+        this.successMarker = successMarker
+    }
+}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
index ab524351274..0d4a51f050a 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
@@ -34,6 +34,7 @@ class PrecommitTasks {
             configureForbiddenApis(project),
             configureCheckstyle(project),
             configureNamingConventions(project),
+            configureLoggerUsage(project),
             project.tasks.create('forbiddenPatterns', ForbiddenPatternsTask.class),
             project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
             project.tasks.create('jarHell', JarHellTask.class),
@@ -63,21 +64,21 @@ class PrecommitTasks {
         project.forbiddenApis {
             internalRuntimeForbidden = true
             failOnUnsupportedJava = false
-            bundledSignatures = ['jdk-unsafe', 'jdk-deprecated']
-            signaturesURLs = [getClass().getResource('/forbidden/all-signatures.txt')]
+            bundledSignatures = ['jdk-unsafe', 'jdk-deprecated', 'jdk-system-out']
+            signaturesURLs = [getClass().getResource('/forbidden/jdk-signatures.txt'),
+                              getClass().getResource('/forbidden/es-all-signatures.txt')]
             suppressAnnotations = ['**.SuppressForbidden']
         }
         Task mainForbidden = project.tasks.findByName('forbiddenApisMain')
         if (mainForbidden != null) {
             mainForbidden.configure {
-                bundledSignatures += 'jdk-system-out'
-                signaturesURLs += getClass().getResource('/forbidden/core-signatures.txt')
+                signaturesURLs += getClass().getResource('/forbidden/es-core-signatures.txt')
             }
         }
         Task testForbidden = project.tasks.findByName('forbiddenApisTest')
         if (testForbidden != null) {
             testForbidden.configure {
-                signaturesURLs += getClass().getResource('/forbidden/test-signatures.txt')
+                signaturesURLs += getClass().getResource('/forbidden/es-test-signatures.txt')
             }
         }
         Task forbiddenApis = project.tasks.findByName('forbiddenApis')
@@ -117,4 +118,18 @@ class PrecommitTasks {
         }
         return null
     }
+
+    private static Task configureLoggerUsage(Project project) {
+        Task loggerUsageTask = project.tasks.create('loggerUsageCheck', LoggerUsageTask.class)
+
+        project.configurations.create('loggerUsagePlugin')
+        project.dependencies.add('loggerUsagePlugin',
+                "org.elasticsearch.test:logger-usage:${org.elasticsearch.gradle.VersionProperties.elasticsearch}")
+
+        loggerUsageTask.configure {
+            classpath = project.configurations.loggerUsagePlugin
+        }
+
+        return loggerUsageTask
+    }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
index c9db5657ba4..19b41cc8cde 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
@@ -23,8 +23,6 @@ import org.gradle.api.Project
 import org.gradle.api.file.FileCollection
 import org.gradle.api.tasks.Input
 
-import java.time.LocalDateTime
-
 /** Configuration for an elasticsearch cluster, used for integration tests. */
 class ClusterConfiguration {
 
@@ -34,6 +32,12 @@ class ClusterConfiguration {
     @Input
     int numNodes = 1
 
+    @Input
+    int numBwcNodes = 0
+
+    @Input
+    String bwcVersion = null
+
     @Input
     int httpPort = 0
 
@@ -49,6 +53,15 @@ class ClusterConfiguration {
     @Input
     String jvmArgs = System.getProperty('tests.jvm.argline', '')
 
+    /**
+     * The seed nodes port file. In the case the cluster has more than one node we use a seed node
+     * to form the cluster. The file is null if there is no seed node yet available.
+     *
+     * Note: this can only be null if the cluster has only one node or if the first node is not yet
+     * configured. All nodes but the first node should see a non null value.
+     */
+    File seedNodePortsFile
+
     /**
      * A closure to call before the cluster is considered ready. The closure is passed the node info,
      * as well as a groovy AntBuilder, to enable running ant condition checks. The default wait
@@ -119,4 +132,12 @@ class ClusterConfiguration {
         }
         extraConfigFiles.put(path, sourceFile)
     }
+
+    /** Returns an address and port suitable for a uri to connect to this clusters seed node over transport protocol*/
+    String seedNodeTransportUri() {
+        if (seedNodePortsFile != null) {
+            return seedNodePortsFile.readLines("UTF-8").get(0)
+        }
+        return null;
+    }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index d96ee511051..a82fefdc510 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -53,13 +53,59 @@ class ClusterFormationTasks {
             // no need to add cluster formation tasks if the task won't run!
             return
         }
-        configureDistributionDependency(project, config.distribution)
-        List<Task> startTasks = []
+        File sharedDir = new File(project.buildDir, "cluster/shared")
+        // first we remove everything in the shared cluster directory to ensure there are no leftovers in repos or anything
+        // in theory this should not be necessary but repositories are only deleted in the cluster-state and not on-disk
+        // such that snapshots survive failures / test runs and there is no simple way today to fix that.
+        Task cleanup = project.tasks.create(name: "${task.name}#prepareCluster.cleanShared", type: Delete, dependsOn: task.dependsOn.collect()) {
+            delete sharedDir
+            doLast {
+                sharedDir.mkdirs()
+            }
+        }
+        List<Task> startTasks = [cleanup]
         List<NodeInfo> nodes = []
+        if (config.numNodes < config.numBwcNodes) {
+            throw new GradleException("numNodes must be >= numBwcNodes [${config.numNodes} < ${config.numBwcNodes}]")
+        }
+        if (config.numBwcNodes > 0 && config.bwcVersion == null) {
+            throw new GradleException("bwcVersion must not be null if numBwcNodes is > 0")
+        }
+        // this is our current version distribution configuration we use for all kinds of REST tests etc.
+        project.configurations {
+            elasticsearchDistro
+        }
+        configureDistributionDependency(project, config.distribution, project.configurations.elasticsearchDistro, VersionProperties.elasticsearch)
+        if (config.bwcVersion != null && config.numBwcNodes > 0) {
+            // if we have a cluster that has a BWC cluster we also need to configure a dependency on the BWC version
+            // this version uses the same distribution etc. and only differs in the version we depend on.
+            // from here on everything else works the same as if it's the current version, we fetch the BWC version
+            // from mirrors using gradles built-in mechanism etc.
+            project.configurations {
+                elasticsearchBwcDistro
+            }
+            configureDistributionDependency(project, config.distribution, project.configurations.elasticsearchBwcDistro, config.bwcVersion)
+        }
+
         for (int i = 0; i < config.numNodes; ++i) {
-            NodeInfo node = new NodeInfo(config, i, project, task)
+            // we start N nodes and out of these N nodes there might be M bwc nodes.
+            // for each of those nodes we might have a different configuratioon
+            String elasticsearchVersion = VersionProperties.elasticsearch
+            Configuration configuration = project.configurations.elasticsearchDistro
+            if (i < config.numBwcNodes) {
+                elasticsearchVersion = config.bwcVersion
+                configuration = project.configurations.elasticsearchBwcDistro
+            }
+            NodeInfo node = new NodeInfo(config, i, project, task, elasticsearchVersion, sharedDir)
+            if (i == 0) {
+                if (config.seedNodePortsFile != null) {
+                    // we might allow this in the future to be set but for now we are the only authority to set this!
+                    throw new GradleException("seedNodePortsFile has a non-null value but first node has not been intialized")
+                }
+                config.seedNodePortsFile = node.transportPortsFile;
+            }
             nodes.add(node)
-            startTasks.add(configureNode(project, task, node))
+            startTasks.add(configureNode(project, task, cleanup, node, configuration))
         }
 
         Task wait = configureWaitTask("${task.name}#wait", project, nodes, startTasks)
@@ -70,20 +116,14 @@ class ClusterFormationTasks {
     }
 
     /** Adds a dependency on the given distribution */
-    static void configureDistributionDependency(Project project, String distro) {
-        String elasticsearchVersion = VersionProperties.elasticsearch
+    static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) {
         String packaging = distro
         if (distro == 'tar') {
             packaging = 'tar.gz'
         } else if (distro == 'integ-test-zip') {
             packaging = 'zip'
         }
-        project.configurations {
-            elasticsearchDistro
-        }
-        project.dependencies {
-            elasticsearchDistro "org.elasticsearch.distribution.${distro}:elasticsearch:${elasticsearchVersion}@${packaging}"
-        }
+        project.dependencies.add(configuration.name, "org.elasticsearch.distribution.${distro}:elasticsearch:${elasticsearchVersion}@${packaging}")
     }
 
     /**
@@ -103,10 +143,10 @@ class ClusterFormationTasks {
      *
      * @return a task which starts the node.
      */
-    static Task configureNode(Project project, Task task, NodeInfo node) {
+    static Task configureNode(Project project, Task task, Object dependsOn, NodeInfo node, Configuration configuration) {
 
         // tasks are chained so their execution order is maintained
-        Task setup = project.tasks.create(name: taskName(task, node, 'clean'), type: Delete, dependsOn: task.dependsOn.collect()) {
+        Task setup = project.tasks.create(name: taskName(task, node, 'clean'), type: Delete, dependsOn: dependsOn) {
             delete node.homeDir
             delete node.cwd
             doLast {
@@ -115,7 +155,7 @@ class ClusterFormationTasks {
         }
         setup = configureCheckPreviousTask(taskName(task, node, 'checkPrevious'), project, setup, node)
         setup = configureStopTask(taskName(task, node, 'stopPrevious'), project, setup, node)
-        setup = configureExtractTask(taskName(task, node, 'extract'), project, setup, node)
+        setup = configureExtractTask(taskName(task, node, 'extract'), project, setup, node, configuration)
         setup = configureWriteConfigTask(taskName(task, node, 'configure'), project, setup, node)
         setup = configureExtraConfigFilesTask(taskName(task, node, 'extraConfig'), project, setup, node)
         setup = configureCopyPluginsTask(taskName(task, node, 'copyPlugins'), project, setup, node)
@@ -151,27 +191,28 @@ class ClusterFormationTasks {
     }
 
     /** Adds a task to extract the elasticsearch distribution */
-    static Task configureExtractTask(String name, Project project, Task setup, NodeInfo node) {
-        List extractDependsOn = [project.configurations.elasticsearchDistro, setup]
-        /* project.configurations.elasticsearchDistro.singleFile will be an
-          external artifact if this is being run by a plugin not living in the
-          elasticsearch source tree. If this is a plugin built in the
-          elasticsearch source tree or this is a distro in the elasticsearch
-          source tree then this should be the version of elasticsearch built
-          by the source tree. If it isn't then Bad Things(TM) will happen. */
+    static Task configureExtractTask(String name, Project project, Task setup, NodeInfo node, Configuration configuration) {
+        List extractDependsOn = [configuration, setup]
+        /* configuration.singleFile will be an external artifact if this is being run by a plugin not living in the
+          elasticsearch source tree. If this is a plugin built in the elasticsearch source tree or this is a distro in
+          the elasticsearch source tree then this should be the version of elasticsearch built by the source tree.
+          If it isn't then Bad Things(TM) will happen. */
         Task extract
+
         switch (node.config.distribution) {
             case 'integ-test-zip':
             case 'zip':
                 extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
-                    from { project.zipTree(project.configurations.elasticsearchDistro.singleFile) }
+                    from {
+                        project.zipTree(configuration.singleFile)
+                    }
                     into node.baseDir
                 }
                 break;
             case 'tar':
                 extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
                     from {
-                        project.tarTree(project.resources.gzip(project.configurations.elasticsearchDistro.singleFile))
+                        project.tarTree(project.resources.gzip(configuration.singleFile))
                     }
                     into node.baseDir
                 }
@@ -180,7 +221,7 @@ class ClusterFormationTasks {
                 File rpmDatabase = new File(node.baseDir, 'rpm-database')
                 File rpmExtracted = new File(node.baseDir, 'rpm-extracted')
                 /* Delay reading the location of the rpm file until task execution */
-                Object rpm = "${ -> project.configurations.elasticsearchDistro.singleFile}"
+                Object rpm = "${ -> configuration.singleFile}"
                 extract = project.tasks.create(name: name, type: LoggedExec, dependsOn: extractDependsOn) {
                     commandLine 'rpm', '--badreloc', '--nodeps', '--noscripts', '--notriggers',
                         '--dbpath', rpmDatabase,
@@ -195,7 +236,7 @@ class ClusterFormationTasks {
             case 'deb':
                 /* Delay reading the location of the deb file until task execution */
                 File debExtracted = new File(node.baseDir, 'deb-extracted')
-                Object deb = "${ -> project.configurations.elasticsearchDistro.singleFile}"
+                Object deb = "${ -> configuration.singleFile}"
                 extract = project.tasks.create(name: name, type: LoggedExec, dependsOn: extractDependsOn) {
                     commandLine 'dpkg-deb', '-x', deb, debExtracted
                     doFirst {
@@ -214,26 +255,28 @@ class ClusterFormationTasks {
         Map esConfig = [
                 'cluster.name'                 : node.clusterName,
                 'pidfile'                      : node.pidFile,
-                'path.repo'                    : "${node.homeDir}/repo",
-                'path.shared_data'             : "${node.homeDir}/../",
+                'path.repo'                    : "${node.sharedDir}/repo",
+                'path.shared_data'             : "${node.sharedDir}/",
                 // Define a node attribute so we can test that it exists
                 'node.testattr'                : 'test',
                 'repositories.url.allowed_urls': 'http://snapshot.test*'
         ]
-        if (node.config.numNodes == 1) {
-            esConfig['http.port'] = node.config.httpPort
-            esConfig['transport.tcp.port'] =  node.config.transportPort
-        } else {
-            // TODO: fix multi node so it doesn't use hardcoded prots
-            esConfig['http.port'] = 9400 + node.nodeNum
-            esConfig['transport.tcp.port'] =  9500 + node.nodeNum
-            esConfig['discovery.zen.ping.unicast.hosts'] = (0..<node.config.numNodes).collect{"localhost:${9500 + it}"}.join(',')
-
-        }
+        esConfig['http.port'] = node.config.httpPort
+        esConfig['transport.tcp.port'] =  node.config.transportPort
         esConfig.putAll(node.config.settings)
 
         Task writeConfig = project.tasks.create(name: name, type: DefaultTask, dependsOn: setup)
         writeConfig.doFirst {
+            if (node.nodeNum > 0) { // multi-node cluster case, we have to wait for the seed node to startup
+                ant.waitfor(maxwait: '20', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond') {
+                    resourceexists {
+                        file(file: node.config.seedNodePortsFile.toString())
+                    }
+                }
+                // the seed node is enough to form the cluster - all subsequent nodes will get the seed node as a unicast
+                // host and join the cluster via that.
+                esConfig['discovery.zen.ping.unicast.hosts'] = "\"${node.config.seedNodeTransportUri()}\""
+            }
             File configFile = new File(node.confDir, 'elasticsearch.yml')
             logger.info("Configuring ${configFile}")
             configFile.setText(esConfig.collect { key, value -> "${key}: ${value}" }.join('\n'), 'UTF-8')
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
index b41b1822000..f68084c61fe 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
@@ -40,6 +40,9 @@ class NodeInfo {
     /** root directory all node files and operations happen under */
     File baseDir
 
+    /** shared data directory all nodes share */
+    File sharedDir
+
     /** the pid file the node will use */
     File pidFile
 
@@ -89,14 +92,15 @@ class NodeInfo {
     ByteArrayOutputStream buffer = new ByteArrayOutputStream()
 
     /** Creates a node to run as part of a cluster for the given task */
-    NodeInfo(ClusterConfiguration config, int nodeNum, Project project, Task task) {
+    NodeInfo(ClusterConfiguration config, int nodeNum, Project project, Task task, String nodeVersion, File sharedDir) {
         this.config = config
         this.nodeNum = nodeNum
+        this.sharedDir = sharedDir
         clusterName = "${task.path.replace(':', '_').substring(1)}"
         baseDir = new File(project.buildDir, "cluster/${task.name} node${nodeNum}")
         pidFile = new File(baseDir, 'es.pid')
-        homeDir = homeDir(baseDir, config.distribution)
-        confDir = confDir(baseDir, config.distribution)
+        homeDir = homeDir(baseDir, config.distribution, nodeVersion)
+        confDir = confDir(baseDir, config.distribution, nodeVersion)
         configFile = new File(confDir, 'elasticsearch.yml')
         // even for rpm/deb, the logs are under home because we dont start with real services
         File logsDir = new File(homeDir, 'logs')
@@ -129,14 +133,15 @@ class NodeInfo {
             'JAVA_HOME' : project.javaHome,
             'ES_GC_OPTS': config.jvmArgs // we pass these with the undocumented gc opts so the argline can set gc, etc
         ]
-        args.add("-Des.node.portsfile=true")
-        args.addAll(config.systemProperties.collect { key, value -> "-D${key}=${value}" })
+        args.addAll("-E", "es.node.portsfile=true")
+        env.put('ES_JAVA_OPTS', config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" "))
         for (Map.Entry<String, String> property : System.properties.entrySet()) {
             if (property.getKey().startsWith('es.')) {
-                args.add("-D${property.getKey()}=${property.getValue()}")
+                args.add("-E")
+                args.add("${property.getKey()}=${property.getValue()}")
             }
         }
-        args.add("-Des.path.conf=${confDir}")
+        args.addAll("-E", "es.path.conf=${confDir}")
         if (Os.isFamily(Os.FAMILY_WINDOWS)) {
             args.add('"') // end the entire command, quoted
         }
@@ -181,13 +186,13 @@ class NodeInfo {
     }
 
     /** Returns the directory elasticsearch home is contained in for the given distribution */
-    static File homeDir(File baseDir, String distro) {
+    static File homeDir(File baseDir, String distro, String nodeVersion) {
         String path
         switch (distro) {
             case 'integ-test-zip':
             case 'zip':
             case 'tar':
-                path = "elasticsearch-${VersionProperties.elasticsearch}"
+                path = "elasticsearch-${nodeVersion}"
                 break
             case 'rpm':
             case 'deb':
@@ -199,12 +204,12 @@ class NodeInfo {
         return new File(baseDir, path)
     }
 
-    static File confDir(File baseDir, String distro) {
+    static File confDir(File baseDir, String distro, String nodeVersion) {
         switch (distro) {
             case 'integ-test-zip':
             case 'zip':
             case 'tar':
-                return new File(homeDir(baseDir, distro), 'config')
+                return new File(homeDir(baseDir, distro, nodeVersion), 'config')
             case 'rpm':
             case 'deb':
                 return new File(baseDir, "${distro}-extracted/etc/elasticsearch")
diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index cbe612e5358..3bce1aece1d 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -208,7 +208,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]TransportClearScrollAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]TransportMultiSearchAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]suggest[/\\]SuggestResponse.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]suggest[/\\]TransportSuggestAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]ActionFilter.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]AutoCreateIndex.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]DelegatingActionListener.java" checks="LineLength" />
@@ -259,7 +258,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]update[/\\]UpdateRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]update[/\\]UpdateRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]Bootstrap.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]BootstrapCLIParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]JNAKernel32Library.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]JNANatives.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]JVMCheck.java" checks="LineLength" />
@@ -269,14 +267,12 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cache[/\\]recycler[/\\]PageCacheRecycler.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]ElasticsearchClient.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]FilterClient.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]Requests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]node[/\\]NodeClient.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]support[/\\]AbstractClient.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]transport[/\\]TransportClient.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]transport[/\\]TransportClientNodesService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]transport[/\\]support[/\\]TransportProxyClient.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterModule.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterState.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterStateObserver.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterStateUpdateTask.java" checks="LineLength" />
@@ -353,16 +349,12 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cli[/\\]CheckFileCommand.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]collect[/\\]ImmutableOpenIntMap.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]geo[/\\]GeoDistance.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]geo[/\\]builders[/\\]LineStringBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]geo[/\\]builders[/\\]PolygonBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]geo[/\\]builders[/\\]ShapeBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]inject[/\\]DefaultConstructionProxyFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]inject[/\\]InjectorImpl.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]inject[/\\]internal[/\\]ConstructionContext.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]inject[/\\]multibindings[/\\]MapBinder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]inject[/\\]spi[/\\]InjectionPoint.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]io[/\\]Channels.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]io[/\\]stream[/\\]NamedWriteableRegistry.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]joda[/\\]Joda.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]lucene[/\\]Lucene.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]lucene[/\\]all[/\\]AllTermQuery.java" checks="LineLength" />
@@ -384,7 +376,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]settings[/\\]IndexScopedSettings.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]settings[/\\]Setting.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]settings[/\\]Settings.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]settings[/\\]loader[/\\]PropertiesSettingsLoader.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]settings[/\\]loader[/\\]XContentSettingsLoader.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]unit[/\\]ByteSizeValue.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]unit[/\\]TimeValue.java" checks="LineLength" />
@@ -393,7 +384,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]CollectionUtils.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]ExtensionPoint.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]concurrent[/\\]EsExecutors.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]concurrent[/\\]EsThreadPoolExecutor.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]concurrent[/\\]PrioritizedEsThreadPoolExecutor.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]concurrent[/\\]ThreadBarrier.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]concurrent[/\\]ThreadContext.java" checks="LineLength" />
@@ -424,7 +414,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]env[/\\]NodeEnvironment.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]AsyncShardFetch.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]DanglingIndicesState.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]Gateway.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayAllocator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayMetaState.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayService.java" checks="LineLength" />
@@ -441,14 +430,12 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]AlreadyExpiredException.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]CompositeIndexEventListener.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexModule.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexSettings.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexingSlowLog.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]MergePolicyConfig.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]MergeSchedulerConfig.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]NodeServicesProvider.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]SearchSlowLog.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]Analysis.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AnalysisRegistry.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AnalysisService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]CommonGramsTokenFilterFactory.java" checks="LineLength" />
@@ -464,7 +451,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]codec[/\\]PerFieldMappingPostingFormatCodec.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]engine[/\\]ElasticsearchConcurrentMergeScheduler.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]engine[/\\]Engine.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]engine[/\\]EngineConfig.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]engine[/\\]InternalEngine.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]engine[/\\]LiveVersionMap.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]engine[/\\]ShadowEngine.java" checks="LineLength" />
@@ -609,7 +595,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]IndexSearcherWrapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]IndexShard.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]IndexingStats.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]ShadowIndexShard.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]ShardPath.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]ShardStateMetaData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]StoreRecovery.java" checks="LineLength" />
@@ -617,7 +602,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]similarity[/\\]SimilarityService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]snapshots[/\\]blobstore[/\\]BlobStoreIndexShardRepository.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]snapshots[/\\]blobstore[/\\]BlobStoreIndexShardSnapshots.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]store[/\\]FsDirectoryService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]store[/\\]IndexStore.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]store[/\\]IndexStoreConfig.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]store[/\\]LegacyVerification.java" checks="LineLength" />
@@ -669,12 +653,8 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]process[/\\]ProcessService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]node[/\\]Node.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]node[/\\]internal[/\\]InternalSettingsPreparer.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolateContext.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolateDocumentParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorQuery.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]DummyPluginInfo.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]InstallPluginCommand.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginsService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]RemovePluginCommand.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]RepositoriesModule.java" checks="LineLength" />
@@ -896,7 +876,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]SuggestUtils.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]Suggesters.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]CompletionSuggestParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]CompletionSuggester.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]context[/\\]CategoryContextMapping.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]context[/\\]ContextMapping.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]context[/\\]GeoContextMapping.java" checks="LineLength" />
@@ -907,12 +886,9 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]LinearInterpoatingScorer.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]NoisyChannelSpellChecker.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]PhraseSuggestParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]PhraseSuggester.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]PhraseSuggestionBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]StupidBackoffScorer.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]WordScorer.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]term[/\\]TermSuggestParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]term[/\\]TermSuggester.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]RestoreService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotInfo.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotShardFailure.java" checks="LineLength" />
@@ -1085,9 +1061,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]FsBlobStoreTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]breaker[/\\]MemoryCircuitBreakerTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]geo[/\\]ShapeBuilderTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]geo[/\\]builders[/\\]AbstractShapeBuilderTestCase.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]geo[/\\]builders[/\\]EnvelopeBuilderTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]geo[/\\]builders[/\\]PolygonBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]hash[/\\]MessageDigestsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]inject[/\\]ModuleTestCase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]io[/\\]stream[/\\]BytesStreamsTests.java" checks="LineLength" />
@@ -1097,8 +1070,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]rounding[/\\]TimeZoneRoundingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]settings[/\\]ScopedSettingsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]settings[/\\]SettingTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]settings[/\\]loader[/\\]JsonSettingsLoaderTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]settings[/\\]loader[/\\]YamlSettingsLoaderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]transport[/\\]BoundTransportAddressTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]unit[/\\]DistanceUnitTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]unit[/\\]FuzzinessTests.java" checks="LineLength" />
@@ -1134,7 +1105,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]fieldstats[/\\]FieldStatsIntegrationIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]fieldstats[/\\]FieldStatsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]AsyncShardFetchTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayIndexStateIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayMetaStateTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayModuleTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayServiceTests.java" checks="LineLength" />
@@ -1155,7 +1125,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]http[/\\]netty[/\\]NettyPipeliningEnabledIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexModuleTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexServiceTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexSettingsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexWithShadowReplicasIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexingSlowLogTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]MergePolicySettingsTests.java" checks="LineLength" />
@@ -1336,14 +1305,9 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]os[/\\]OsProbeTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]nodesinfo[/\\]NodeInfoStreamingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]options[/\\]detailederrors[/\\]DetailedErrorsEnabledIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]ConcurrentPercolatorIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]MultiPercolatorIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolateDocumentParserTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorQueryTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorServiceTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]RecoveryPercolatorIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]TTLPercolatorIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginInfoTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginsServiceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]recovery[/\\]FullRollingRestartIT.java" checks="LineLength" />
@@ -1403,7 +1367,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]basic[/\\]SearchWithRandomIOExceptionsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]basic[/\\]TransportSearchFailuresIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]basic[/\\]TransportTwoNodesSearchIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]builder[/\\]SearchSourceBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]child[/\\]ChildQuerySearchIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]child[/\\]ParentFieldLoadingIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]fetch[/\\]FetchSubPhasePluginIT.java" checks="LineLength" />
@@ -1425,7 +1388,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]query[/\\]ExistsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]query[/\\]MultiMatchQueryIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]query[/\\]SearchQueryIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]rescore[/\\]QueryRescoreBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]scroll[/\\]DuelScrollIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]scroll[/\\]SearchScrollIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]scroll[/\\]SearchScrollWithFailingNodesIT.java" checks="LineLength" />
@@ -1436,12 +1398,10 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]CompletionSuggestSearchIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]ContextCompletionSuggestSearchIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]CustomSuggester.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]CustomSuggesterSearchIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]CategoryContextMappingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]GeoContextMappingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]DirectCandidateGeneratorTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]NoisyChannelSpellCheckerTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]SmoothingModelTestCase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]similarity[/\\]SimilarityIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]AbstractSnapshotIntegTestCase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]BlobStoreFormatIT.java" checks="LineLength" />
@@ -1597,7 +1557,6 @@
   <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]aws[/\\]blobstore[/\\]MockDefaultS3OutputStream.java" checks="LineLength" />
   <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]s3[/\\]AbstractS3SnapshotRestoreTest.java" checks="LineLength" />
   <suppress files="plugins[/\\]store-smb[/\\]src[/\\]main[/\\]java[/\\]org[/\\]apache[/\\]lucene[/\\]store[/\\]SmbDirectoryWrapper.java" checks="LineLength" />
-  <suppress files="qa[/\\]evil-tests[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]BootstrapCliParserTests.java" checks="LineLength" />
   <suppress files="qa[/\\]evil-tests[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]ESPolicyUnitTests.java" checks="LineLength" />
   <suppress files="qa[/\\]evil-tests[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]EvilSecurityTests.java" checks="LineLength" />
   <suppress files="qa[/\\]evil-tests[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cli[/\\]CheckFileCommandTests.java" checks="LineLength" />
diff --git a/buildSrc/src/main/resources/forbidden/es-all-signatures.txt b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt
new file mode 100644
index 00000000000..d258c098911
--- /dev/null
+++ b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt
@@ -0,0 +1,30 @@
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance  with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on
+# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+java.nio.file.Paths @ Use org.elasticsearch.common.io.PathUtils.get() instead.
+java.nio.file.FileSystems#getDefault() @ use org.elasticsearch.common.io.PathUtils.getDefaultFileSystem() instead.
+
+java.nio.file.Files#getFileStore(java.nio.file.Path) @ Use org.elasticsearch.env.Environment.getFileStore() instead, impacted by JDK-8034057
+java.nio.file.Files#isWritable(java.nio.file.Path) @ Use org.elasticsearch.env.Environment.isWritable() instead, impacted by JDK-8034057
+
+@defaultMessage Use org.elasticsearch.common.Randomness#get for reproducible sources of randomness
+java.util.Random#<init>()
+java.util.concurrent.ThreadLocalRandom
+
+java.security.MessageDigest#clone() @ use org.elasticsearch.common.hash.MessageDigests
+
+@defaultMessage this should not have been added to lucene in the first place
+org.apache.lucene.index.IndexReader#getCombinedCoreAndDeletesKey()
diff --git a/buildSrc/src/main/resources/forbidden/core-signatures.txt b/buildSrc/src/main/resources/forbidden/es-core-signatures.txt
similarity index 100%
rename from buildSrc/src/main/resources/forbidden/core-signatures.txt
rename to buildSrc/src/main/resources/forbidden/es-core-signatures.txt
diff --git a/buildSrc/src/main/resources/forbidden/test-signatures.txt b/buildSrc/src/main/resources/forbidden/es-test-signatures.txt
similarity index 100%
rename from buildSrc/src/main/resources/forbidden/test-signatures.txt
rename to buildSrc/src/main/resources/forbidden/es-test-signatures.txt
diff --git a/buildSrc/src/main/resources/forbidden/all-signatures.txt b/buildSrc/src/main/resources/forbidden/jdk-signatures.txt
similarity index 85%
rename from buildSrc/src/main/resources/forbidden/all-signatures.txt
rename to buildSrc/src/main/resources/forbidden/jdk-signatures.txt
index 9bc37005514..994b1ad3a4a 100644
--- a/buildSrc/src/main/resources/forbidden/all-signatures.txt
+++ b/buildSrc/src/main/resources/forbidden/jdk-signatures.txt
@@ -33,9 +33,6 @@ java.util.Formatter#<init>(java.lang.String,java.lang.String,java.util.Locale)
 java.io.RandomAccessFile
 java.nio.file.Path#toFile()
 
-java.nio.file.Paths @ Use org.elasticsearch.common.io.PathUtils.get() instead.
-java.nio.file.FileSystems#getDefault() @ use org.elasticsearch.common.io.PathUtils.getDefaultFileSystem() instead.
-
 @defaultMessage Specify a location for the temp file/directory instead.
 java.nio.file.Files#createTempDirectory(java.lang.String,java.nio.file.attribute.FileAttribute[])
 java.nio.file.Files#createTempFile(java.lang.String,java.lang.String,java.nio.file.attribute.FileAttribute[])
@@ -48,9 +45,6 @@ java.io.ObjectInput
 
 java.nio.file.Files#isHidden(java.nio.file.Path) @ Dependent on the operating system, use FileSystemUtils.isHidden instead
 
-java.nio.file.Files#getFileStore(java.nio.file.Path) @ Use org.elasticsearch.env.Environment.getFileStore() instead, impacted by JDK-8034057
-java.nio.file.Files#isWritable(java.nio.file.Path) @ Use org.elasticsearch.env.Environment.isWritable() instead, impacted by JDK-8034057
-
 @defaultMessage Resolve hosts explicitly to the address(es) you want with InetAddress.
 java.net.InetSocketAddress#<init>(java.lang.String,int)
 java.net.Socket#<init>(java.lang.String,int)
@@ -89,9 +83,6 @@ java.lang.Class#getDeclaredMethods() @ Do not violate java's access system: Use
 java.lang.reflect.AccessibleObject#setAccessible(boolean)
 java.lang.reflect.AccessibleObject#setAccessible(java.lang.reflect.AccessibleObject[], boolean)
 
-@defaultMessage this should not have been added to lucene in the first place
-org.apache.lucene.index.IndexReader#getCombinedCoreAndDeletesKey()
-
 @defaultMessage this method needs special permission
 java.lang.Thread#getAllStackTraces()
 
@@ -112,8 +103,3 @@ java.util.Collections#EMPTY_MAP
 java.util.Collections#EMPTY_SET
 
 java.util.Collections#shuffle(java.util.List) @ Use java.util.Collections#shuffle(java.util.List, java.util.Random) with a reproducible source of randomness
-@defaultMessage Use org.elasticsearch.common.Randomness#get for reproducible sources of randomness
-java.util.Random#<init>()
-java.util.concurrent.ThreadLocalRandom
-
-java.security.MessageDigest#clone() @ use org.elasticsearch.common.hash.MessageDigests
diff --git a/buildSrc/src/main/resources/plugin-descriptor.properties b/buildSrc/src/main/resources/plugin-descriptor.properties
index 7659f11bd09..ebde46d326b 100644
--- a/buildSrc/src/main/resources/plugin-descriptor.properties
+++ b/buildSrc/src/main/resources/plugin-descriptor.properties
@@ -38,12 +38,3 @@ java.version=${javaVersion}
 #
 # 'elasticsearch.version' version of elasticsearch compiled against
 elasticsearch.version=${elasticsearchVersion}
-#
-### deprecated elements for jvm plugins :
-#
-# 'isolated': true if the plugin should have its own classloader.
-# passing false is deprecated, and only intended to support plugins
-# that have hard dependencies against each other. If this is
-# not specified, then the plugin is isolated by default.
-isolated=${isolated}
-#
\ No newline at end of file
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index f75d5a936bb..13c0885a61a 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,5 +1,5 @@
-elasticsearch     = 5.0.0
-lucene            = 6.0.0-snapshot-bea235f
+elasticsearch     = 5.0.0-alpha1
+lucene            = 6.0.0-snapshot-f0aa4fc
 
 # optional dependencies
 spatial4j         = 0.6
diff --git a/core/build.gradle b/core/build.gradle
index 39c1e4367c0..ab3754e72ff 100644
--- a/core/build.gradle
+++ b/core/build.gradle
@@ -49,7 +49,7 @@ dependencies {
   compile 'org.elasticsearch:securesm:1.0'
 
   // utilities
-  compile 'commons-cli:commons-cli:1.3.1'
+  compile 'net.sf.jopt-simple:jopt-simple:4.9'
   compile 'com.carrotsearch:hppc:0.7.1'
 
   // time handling, remove with java 8 time
diff --git a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java
index 4e24944ffac..564f780b8ed 100644
--- a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java
+++ b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java
@@ -237,6 +237,10 @@ public abstract class BlendedTermQuery extends Query {
         return newCtx;
     }
 
+    public List<Term> getTerms() {
+        return Arrays.asList(terms);
+    }
+
     @Override
     public String toString(String field) {
         StringBuilder builder = new StringBuilder("blended(terms:[");
diff --git a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java
index a7c53a56bc4..6ddd7591caa 100644
--- a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java
+++ b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java
@@ -787,8 +787,9 @@ public class MapperQueryParser extends QueryParser {
             assert q instanceof BoostQuery == false;
             return pq;
         } else if (q instanceof MultiPhraseQuery) {
-            ((MultiPhraseQuery) q).setSlop(slop);
-            return q;
+            MultiPhraseQuery.Builder builder = new MultiPhraseQuery.Builder((MultiPhraseQuery) q);
+            builder.setSlop(slop);
+            return builder.build();
         } else {
             return q;
         }
diff --git a/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java b/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java
index 089b649cefe..3c0bda97347 100644
--- a/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java
+++ b/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java
@@ -34,7 +34,6 @@ import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 
 import java.io.IOException;
 import java.util.Collection;
-import java.util.List;
 
 /**
  *
@@ -68,7 +67,7 @@ public class CustomFieldQuery extends FieldQuery {
             flatten(((FiltersFunctionScoreQuery) sourceQuery).getSubQuery(), reader, flatQueries, boost);
         } else if (sourceQuery instanceof MultiPhraseQuery) {
             MultiPhraseQuery q = ((MultiPhraseQuery) sourceQuery);
-            convertMultiPhraseQuery(0, new int[q.getTermArrays().size()], q, q.getTermArrays(), q.getPositions(), reader, flatQueries);
+            convertMultiPhraseQuery(0, new int[q.getTermArrays().length], q, q.getTermArrays(), q.getPositions(), reader, flatQueries);
         } else if (sourceQuery instanceof BlendedTermQuery) {
             final BlendedTermQuery blendedTermQuery = (BlendedTermQuery) sourceQuery;
             flatten(blendedTermQuery.rewrite(reader), reader, flatQueries, boost);
@@ -77,7 +76,7 @@ public class CustomFieldQuery extends FieldQuery {
         }
     }
 
-    private void convertMultiPhraseQuery(int currentPos, int[] termsIdx, MultiPhraseQuery orig, List<Term[]> terms, int[] pos, IndexReader reader, Collection<Query> flatQueries) throws IOException {
+    private void convertMultiPhraseQuery(int currentPos, int[] termsIdx, MultiPhraseQuery orig, Term[][] terms, int[] pos, IndexReader reader, Collection<Query> flatQueries) throws IOException {
         if (currentPos == 0) {
             // if we have more than 16 terms
             int numTerms = 0;
@@ -97,16 +96,16 @@ public class CustomFieldQuery extends FieldQuery {
          * we walk all possible ways and for each path down the MPQ we create a PhraseQuery this is what FieldQuery supports.
          * It seems expensive but most queries will pretty small.
          */
-        if (currentPos == terms.size()) {
+        if (currentPos == terms.length) {
             PhraseQuery.Builder queryBuilder = new PhraseQuery.Builder();
             queryBuilder.setSlop(orig.getSlop());
             for (int i = 0; i < termsIdx.length; i++) {
-                queryBuilder.add(terms.get(i)[termsIdx[i]], pos[i]);
+                queryBuilder.add(terms[i][termsIdx[i]], pos[i]);
             }
             Query query = queryBuilder.build();
             this.flatten(query, reader, flatQueries, 1F);
         } else {
-            Term[] t = terms.get(currentPos);
+            Term[] t = terms[currentPos];
             for (int i = 0; i < t.length; i++) {
                 termsIdx[currentPos] = i;
                 convertMultiPhraseQuery(currentPos+1, termsIdx, orig, terms, pos, reader, flatQueries);
diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchException.java b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
index d069bddfdfe..7fd81f5ddfe 100644
--- a/core/src/main/java/org/elasticsearch/ElasticsearchException.java
+++ b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
@@ -645,8 +645,6 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
         // 87 used to be for MergeMappingException
         INVALID_INDEX_TEMPLATE_EXCEPTION(org.elasticsearch.indices.InvalidIndexTemplateException.class,
                 org.elasticsearch.indices.InvalidIndexTemplateException::new, 88),
-        PERCOLATE_EXCEPTION(org.elasticsearch.percolator.PercolateException.class,
-                org.elasticsearch.percolator.PercolateException::new, 89),
         REFRESH_FAILED_ENGINE_EXCEPTION(org.elasticsearch.index.engine.RefreshFailedEngineException.class,
                 org.elasticsearch.index.engine.RefreshFailedEngineException::new, 90),
         AGGREGATION_INITIALIZATION_EXCEPTION(org.elasticsearch.search.aggregations.AggregationInitializationException.class,
diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index eeb4825cb90..fe794d96661 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -60,11 +60,13 @@ public class Version {
     public static final Version V_2_1_2 = new Version(V_2_1_2_ID, org.apache.lucene.util.Version.LUCENE_5_3_1);
     public static final int V_2_2_0_ID = 2020099;
     public static final Version V_2_2_0 = new Version(V_2_2_0_ID, org.apache.lucene.util.Version.LUCENE_5_4_1);
+    public static final int V_2_2_1_ID = 2020199;
+    public static final Version V_2_2_1 = new Version(V_2_2_1_ID, org.apache.lucene.util.Version.LUCENE_5_4_1);
     public static final int V_2_3_0_ID = 2030099;
     public static final Version V_2_3_0 = new Version(V_2_3_0_ID, org.apache.lucene.util.Version.LUCENE_5_5_0);
-    public static final int V_5_0_0_ID = 5000099;
-    public static final Version V_5_0_0 = new Version(V_5_0_0_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
-    public static final Version CURRENT = V_5_0_0;
+    public static final int V_5_0_0_alpha1_ID = 5000001;
+    public static final Version V_5_0_0_alpha1 = new Version(V_5_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
+    public static final Version CURRENT = V_5_0_0_alpha1;
 
     static {
         assert CURRENT.luceneVersion.equals(org.apache.lucene.util.Version.LATEST) : "Version must be upgraded to ["
@@ -77,10 +79,12 @@ public class Version {
 
     public static Version fromId(int id) {
         switch (id) {
-            case V_5_0_0_ID:
-                return V_5_0_0;
+            case V_5_0_0_alpha1_ID:
+                return V_5_0_0_alpha1;
             case V_2_3_0_ID:
                 return V_2_3_0;
+            case V_2_2_1_ID:
+                return V_2_2_1;
             case V_2_2_0_ID:
                 return V_2_2_0;
             case V_2_1_2_ID:
diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java
index 491202e7c7a..2b33a669428 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -165,7 +165,6 @@ import org.elasticsearch.action.percolate.MultiPercolateAction;
 import org.elasticsearch.action.percolate.PercolateAction;
 import org.elasticsearch.action.percolate.TransportMultiPercolateAction;
 import org.elasticsearch.action.percolate.TransportPercolateAction;
-import org.elasticsearch.action.percolate.TransportShardMultiPercolateAction;
 import org.elasticsearch.action.search.ClearScrollAction;
 import org.elasticsearch.action.search.MultiSearchAction;
 import org.elasticsearch.action.search.SearchAction;
@@ -174,8 +173,6 @@ import org.elasticsearch.action.search.TransportClearScrollAction;
 import org.elasticsearch.action.search.TransportMultiSearchAction;
 import org.elasticsearch.action.search.TransportSearchAction;
 import org.elasticsearch.action.search.TransportSearchScrollAction;
-import org.elasticsearch.action.suggest.SuggestAction;
-import org.elasticsearch.action.suggest.TransportSuggestAction;
 import org.elasticsearch.action.support.ActionFilter;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.AutoCreateIndex;
@@ -321,7 +318,6 @@ public class ActionModule extends AbstractModule {
         registerAction(MultiTermVectorsAction.INSTANCE, TransportMultiTermVectorsAction.class,
                 TransportShardMultiTermsVectorAction.class);
         registerAction(DeleteAction.INSTANCE, TransportDeleteAction.class);
-        registerAction(SuggestAction.INSTANCE, TransportSuggestAction.class);
         registerAction(UpdateAction.INSTANCE, TransportUpdateAction.class);
         registerAction(MultiGetAction.INSTANCE, TransportMultiGetAction.class,
                 TransportShardMultiGetAction.class);
@@ -331,7 +327,7 @@ public class ActionModule extends AbstractModule {
         registerAction(SearchScrollAction.INSTANCE, TransportSearchScrollAction.class);
         registerAction(MultiSearchAction.INSTANCE, TransportMultiSearchAction.class);
         registerAction(PercolateAction.INSTANCE, TransportPercolateAction.class);
-        registerAction(MultiPercolateAction.INSTANCE, TransportMultiPercolateAction.class, TransportShardMultiPercolateAction.class);
+        registerAction(MultiPercolateAction.INSTANCE, TransportMultiPercolateAction.class);
         registerAction(ExplainAction.INSTANCE, TransportExplainAction.class);
         registerAction(ClearScrollAction.INSTANCE, TransportClearScrollAction.class);
         registerAction(RecoveryAction.INSTANCE, TransportRecoveryAction.class);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
index b5c9577aff7..069f0ebe1b8 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
@@ -32,6 +31,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -213,7 +213,7 @@ public class TransportClusterHealthAction extends TransportMasterNodeReadAction<
         }
         if (request.indices() != null && request.indices().length > 0) {
             try {
-                indexNameExpressionResolver.concreteIndices(clusterState, IndicesOptions.strictExpand(), request.indices());
+                indexNameExpressionResolver.concreteIndexNames(clusterState, IndicesOptions.strictExpand(), request.indices());
                 waitForCounter++;
             } catch (IndexNotFoundException e) {
                 response.setStatus(ClusterHealthStatus.RED); // no indices, make sure its RED
@@ -280,7 +280,7 @@ public class TransportClusterHealthAction extends TransportMasterNodeReadAction<
 
         String[] concreteIndices;
         try {
-            concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, request);
+            concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request);
         } catch (IndexNotFoundException e) {
             // one of the specified indices is not there - treat it as RED.
             ClusterHealthResponse response = new ClusterHealthResponse(clusterName.value(), Strings.EMPTY_ARRAY, clusterState,
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
index c743a1d2a91..d53f651da45 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
@@ -24,8 +24,8 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
index 11c542863b5..87ec2d052ab 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
@@ -231,8 +231,7 @@ public class NodeInfo extends BaseNodeResponse {
             plugins.readFrom(in);
         }
         if (in.readBoolean()) {
-            ingest = new IngestInfo();
-            ingest.readFrom(in);
+            ingest = new IngestInfo(in);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
index b14450f9eb1..f52729faa4f 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
@@ -23,8 +23,8 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/TransportLivenessAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/TransportLivenessAction.java
index 65913bc4b28..8b29b9379ac 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/TransportLivenessAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/TransportLivenessAction.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.action.admin.cluster.node.liveness;
 
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportChannel;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java
index a4cf2b1de2a..442b5edde77 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java
@@ -31,6 +31,7 @@ import org.elasticsearch.discovery.DiscoveryStats;
 import org.elasticsearch.http.HttpStats;
 import org.elasticsearch.indices.NodeIndicesStats;
 import org.elasticsearch.indices.breaker.AllCircuitBreakerStats;
+import org.elasticsearch.ingest.IngestStats;
 import org.elasticsearch.monitor.fs.FsInfo;
 import org.elasticsearch.monitor.jvm.JvmStats;
 import org.elasticsearch.monitor.os.OsStats;
@@ -81,6 +82,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContent {
     @Nullable
     private DiscoveryStats discoveryStats;
 
+    @Nullable
+    private IngestStats ingestStats;
+
     NodeStats() {
     }
 
@@ -89,7 +93,8 @@ public class NodeStats extends BaseNodeResponse implements ToXContent {
                      @Nullable FsInfo fs, @Nullable TransportStats transport, @Nullable HttpStats http,
                      @Nullable AllCircuitBreakerStats breaker,
                      @Nullable ScriptStats scriptStats,
-                     @Nullable DiscoveryStats discoveryStats) {
+                     @Nullable DiscoveryStats discoveryStats,
+                     @Nullable IngestStats ingestStats) {
         super(node);
         this.timestamp = timestamp;
         this.indices = indices;
@@ -103,6 +108,7 @@ public class NodeStats extends BaseNodeResponse implements ToXContent {
         this.breaker = breaker;
         this.scriptStats = scriptStats;
         this.discoveryStats = discoveryStats;
+        this.ingestStats = ingestStats;
     }
 
     public long getTimestamp() {
@@ -187,6 +193,11 @@ public class NodeStats extends BaseNodeResponse implements ToXContent {
         return this.discoveryStats;
     }
 
+    @Nullable
+    public IngestStats getIngestStats() {
+        return ingestStats;
+    }
+
     public static NodeStats readNodeStats(StreamInput in) throws IOException {
         NodeStats nodeInfo = new NodeStats();
         nodeInfo.readFrom(in);
@@ -224,7 +235,7 @@ public class NodeStats extends BaseNodeResponse implements ToXContent {
         breaker = AllCircuitBreakerStats.readOptionalAllCircuitBreakerStats(in);
         scriptStats = in.readOptionalStreamable(ScriptStats::new);
         discoveryStats = in.readOptionalStreamable(() -> new DiscoveryStats(null));
-
+        ingestStats = in.readOptionalWriteable(IngestStats::new);
     }
 
     @Override
@@ -282,6 +293,7 @@ public class NodeStats extends BaseNodeResponse implements ToXContent {
         out.writeOptionalStreamable(breaker);
         out.writeOptionalStreamable(scriptStats);
         out.writeOptionalStreamable(discoveryStats);
+        out.writeOptionalWriteable(ingestStats);
     }
 
     @Override
@@ -337,6 +349,10 @@ public class NodeStats extends BaseNodeResponse implements ToXContent {
             getDiscoveryStats().toXContent(builder, params);
         }
 
+        if (getIngestStats() != null) {
+            getIngestStats().toXContent(builder, params);
+        }
+
         return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java
index 5916421c1ed..88162a617a8 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java
@@ -42,6 +42,7 @@ public class NodesStatsRequest extends BaseNodesRequest<NodesStatsRequest> {
     private boolean breaker;
     private boolean script;
     private boolean discovery;
+    private boolean ingest;
 
     public NodesStatsRequest() {
     }
@@ -69,6 +70,7 @@ public class NodesStatsRequest extends BaseNodesRequest<NodesStatsRequest> {
         this.breaker = true;
         this.script = true;
         this.discovery = true;
+        this.ingest = true;
         return this;
     }
 
@@ -87,6 +89,7 @@ public class NodesStatsRequest extends BaseNodesRequest<NodesStatsRequest> {
         this.breaker = false;
         this.script = false;
         this.discovery = false;
+        this.ingest = false;
         return this;
     }
 
@@ -250,6 +253,17 @@ public class NodesStatsRequest extends BaseNodesRequest<NodesStatsRequest> {
         return this;
     }
 
+    public boolean ingest() {
+        return ingest;
+    }
+
+    /**
+     * Should ingest statistics be returned.
+     */
+    public NodesStatsRequest ingest(boolean ingest) {
+        this.ingest = ingest;
+        return this;
+    }
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
@@ -265,6 +279,7 @@ public class NodesStatsRequest extends BaseNodesRequest<NodesStatsRequest> {
         breaker = in.readBoolean();
         script = in.readBoolean();
         discovery = in.readBoolean();
+        ingest = in.readBoolean();
     }
 
     @Override
@@ -281,6 +296,7 @@ public class NodesStatsRequest extends BaseNodesRequest<NodesStatsRequest> {
         out.writeBoolean(breaker);
         out.writeBoolean(script);
         out.writeBoolean(discovery);
+        out.writeBoolean(ingest);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java
index dc35eefee7d..027e6122681 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java
@@ -137,4 +137,12 @@ public class NodesStatsRequestBuilder extends NodesOperationRequestBuilder<Nodes
         request.discovery(discovery);
         return this;
     }
+
+    /**
+     * Should ingest statistics be returned.
+     */
+    public NodesStatsRequestBuilder ingest(boolean ingest) {
+        request.ingest(ingest);
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
index 8460eb59ba2..8ba3d00558b 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
@@ -23,8 +23,8 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -80,7 +80,8 @@ public class TransportNodesStatsAction extends TransportNodesAction<NodesStatsRe
     protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) {
         NodesStatsRequest request = nodeStatsRequest.request;
         return nodeService.stats(request.indices(), request.os(), request.process(), request.jvm(), request.threadPool(),
-                request.fs(), request.transport(), request.http(), request.breaker(), request.script(), request.discovery());
+                request.fs(), request.transport(), request.http(), request.breaker(), request.script(), request.discovery(),
+                request.ingest());
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequest.java
index 1d1249e1551..e92695d61e2 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksRequest.java
@@ -53,12 +53,18 @@ public class CancelTasksRequest extends BaseTasksRequest<CancelTasksRequest> {
         return super.match(task) && task instanceof CancellableTask;
     }
 
-    public CancelTasksRequest reason(String reason) {
+    /**
+     * Set the reason for canceling the task.
+     */
+    public CancelTasksRequest setReason(String reason) {
         this.reason = reason;
         return this;
     }
 
-    public String reason() {
+    /**
+     * The reason for canceling the task.
+     */
+    public String getReason() {
         return reason;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
index b07e540d792..9dbe4ee1aeb 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
@@ -26,10 +26,10 @@ import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -84,21 +84,21 @@ public class TransportCancelTasksAction extends TransportTasksAction<Cancellable
     }
 
     protected void processTasks(CancelTasksRequest request, Consumer<CancellableTask> operation) {
-        if (request.taskId().isSet() == false) {
+        if (request.getTaskId().isSet()) {
             // we are only checking one task, we can optimize it
-            CancellableTask task = taskManager.getCancellableTask(request.taskId().getId());
+            CancellableTask task = taskManager.getCancellableTask(request.getTaskId().getId());
             if (task != null) {
                 if (request.match(task)) {
                     operation.accept(task);
                 } else {
-                    throw new IllegalArgumentException("task [" + request.taskId() + "] doesn't support this operation");
+                    throw new IllegalArgumentException("task [" + request.getTaskId() + "] doesn't support this operation");
                 }
             } else {
-                if (taskManager.getTask(request.taskId().getId()) != null) {
+                if (taskManager.getTask(request.getTaskId().getId()) != null) {
                     // The task exists, but doesn't support cancellation
-                    throw new IllegalArgumentException("task [" + request.taskId() + "] doesn't support cancellation");
+                    throw new IllegalArgumentException("task [" + request.getTaskId() + "] doesn't support cancellation");
                 } else {
-                    throw new ResourceNotFoundException("task [{}] doesn't support cancellation", request.taskId());
+                    throw new ResourceNotFoundException("task [{}] doesn't support cancellation", request.getTaskId());
                 }
             }
         } else {
@@ -113,14 +113,14 @@ public class TransportCancelTasksAction extends TransportTasksAction<Cancellable
     @Override
     protected synchronized TaskInfo taskOperation(CancelTasksRequest request, CancellableTask cancellableTask) {
         final BanLock banLock = new BanLock(nodes -> removeBanOnNodes(cancellableTask, nodes));
-        Set<String> childNodes = taskManager.cancel(cancellableTask, request.reason(), banLock::onTaskFinished);
+        Set<String> childNodes = taskManager.cancel(cancellableTask, request.getReason(), banLock::onTaskFinished);
         if (childNodes != null) {
             if (childNodes.isEmpty()) {
                 logger.trace("cancelling task {} with no children", cancellableTask.getId());
                 return cancellableTask.taskInfo(clusterService.localNode(), false);
             } else {
                 logger.trace("cancelling task {} with children on nodes [{}]", cancellableTask.getId(), childNodes);
-                setBanOnNodes(request.reason(), cancellableTask, childNodes, banLock);
+                setBanOnNodes(request.getReason(), cancellableTask, childNodes, banLock);
                 return cancellableTask.taskInfo(clusterService.localNode(), false);
             }
         } else {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java
index 6bf8ac3e1ef..3fe743fc36a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java
@@ -31,31 +31,49 @@ import java.io.IOException;
 public class ListTasksRequest extends BaseTasksRequest<ListTasksRequest> {
 
     private boolean detailed = false;
+    private boolean waitForCompletion = false;
 
     /**
      * Should the detailed task information be returned.
      */
-    public boolean detailed() {
+    public boolean getDetailed() {
         return this.detailed;
     }
 
     /**
      * Should the detailed task information be returned.
      */
-    public ListTasksRequest detailed(boolean detailed) {
+    public ListTasksRequest setDetailed(boolean detailed) {
         this.detailed = detailed;
         return this;
     }
 
+    /**
+     * Should this request wait for all found tasks to complete?
+     */
+    public boolean getWaitForCompletion() {
+        return waitForCompletion;
+    }
+
+    /**
+     * Should this request wait for all found tasks to complete?
+     */
+    public ListTasksRequest setWaitForCompletion(boolean waitForCompletion) {
+        this.waitForCompletion = waitForCompletion;
+        return this;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         detailed = in.readBoolean();
+        waitForCompletion = in.readBoolean();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeBoolean(detailed);
+        out.writeBoolean(waitForCompletion);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequestBuilder.java
index 2b462014f43..1385781125a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequestBuilder.java
@@ -35,7 +35,15 @@ public class ListTasksRequestBuilder extends TasksRequestBuilder<ListTasksReques
      * Should detailed task information be returned.
      */
     public ListTasksRequestBuilder setDetailed(boolean detailed) {
-        request.detailed(detailed);
+        request.setDetailed(detailed);
+        return this;
+    }
+
+    /**
+     * Should this request wait for all found tasks to complete?
+     */
+    public final ListTasksRequestBuilder setWaitForCompletion(boolean waitForCompletion) {
+        request.setWaitForCompletion(waitForCompletion);
         return this;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java
index c027bfa7ab4..ad9fa9509e8 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java
@@ -178,7 +178,7 @@ public class TaskInfo implements Writeable<TaskInfo>, ToXContent {
         }
         builder.dateValueField("start_time_in_millis", "start_time", startTime);
         builder.timeValueField("running_time_in_nanos", "running_time", runningTimeNanos, TimeUnit.NANOSECONDS);
-        if (parentTaskId.isSet() == false) {
+        if (parentTaskId.isSet()) {
             builder.field("parent_task_id", parentTaskId.toString());
         }
         return builder;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
index 40f8894d3be..e6ea002a794 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
@@ -19,28 +19,36 @@
 
 package org.elasticsearch.action.admin.cluster.node.tasks.list;
 
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.List;
+import java.util.function.Consumer;
+
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 
 /**
  *
  */
 public class TransportListTasksAction extends TransportTasksAction<Task, ListTasksRequest, ListTasksResponse, TaskInfo> {
+    private static final TimeValue WAIT_FOR_COMPLETION_POLL = timeValueMillis(100);
+    private static final TimeValue DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT = timeValueSeconds(30);
 
     @Inject
     public TransportListTasksAction(Settings settings, ClusterName clusterName, ThreadPool threadPool, ClusterService clusterService, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
@@ -59,7 +67,40 @@ public class TransportListTasksAction extends TransportTasksAction<Task, ListTas
 
     @Override
     protected TaskInfo taskOperation(ListTasksRequest request, Task task) {
-        return task.taskInfo(clusterService.localNode(), request.detailed());
+        return task.taskInfo(clusterService.localNode(), request.getDetailed());
+    }
+
+    @Override
+    protected void processTasks(ListTasksRequest request, Consumer<Task> operation) {
+        if (false == request.getWaitForCompletion()) {
+            super.processTasks(request, operation);
+            return;
+        }
+        // If we should wait for completion then we have to intercept every found task and wait for it to leave the manager.
+        TimeValue timeout = request.getTimeout();
+        if (timeout == null) {
+            timeout = DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT;
+        }
+        long timeoutTime = System.nanoTime() + timeout.nanos();
+        super.processTasks(request, operation.andThen((Task t) -> {
+            while (System.nanoTime() - timeoutTime < 0) {
+                Task task = taskManager.getTask(t.getId());
+                if (task == null) {
+                    return;
+                }
+                if (task.getAction().startsWith(ListTasksAction.NAME)) {
+                    // It doesn't make sense to wait for List Tasks and it can cause an infinite loop of the task waiting
+                    // for itself of one of its child tasks
+                    return;
+                }
+                try {
+                    Thread.sleep(WAIT_FOR_COMPLETION_POLL.millis());
+                } catch (InterruptedException e) {
+                    throw new ElasticsearchException("Interrupted waiting for completion of [{}]", e, t);
+                }
+            }
+            throw new ElasticsearchTimeoutException("Timed out waiting for completion of [{}]", t);
+        }));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java
index 79e51f9a46e..a17d2aac892 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java
@@ -22,12 +22,12 @@ package org.elasticsearch.action.admin.cluster.repositories.delete;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.repositories.RepositoriesService;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java
index 39d9cacbda3..490d20f086c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java
@@ -22,7 +22,6 @@ package org.elasticsearch.action.admin.cluster.repositories.get;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
@@ -30,6 +29,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java
index efc45f16cbd..d1639001352 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java
@@ -22,12 +22,12 @@ package org.elasticsearch.action.admin.cluster.repositories.put;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.repositories.RepositoriesService;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java
index 8b1d9816004..2c75335dcaa 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java
@@ -23,11 +23,11 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.repositories.RepositoriesService;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
index d7ec84fb7a5..e6116dbfbc4 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
@@ -31,6 +30,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.RoutingExplanations;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
index 75f94921e61..60a0e7a8046 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
@@ -33,6 +32,7 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.inject.Inject;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java
index 9f3ccac8f64..1b329d17289 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java
@@ -67,7 +67,7 @@ public class ClusterSearchShardsGroup implements Streamable, ToXContent {
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
-        index = Index.readIndex(in);
+        index = new Index(in);
         shardId = in.readVInt();
         shards = new ShardRouting[in.readVInt()];
         for (int i = 0; i < shards.length; i++) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java
index e6a9d98eb17..8b26fd6c04f 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java
@@ -22,7 +22,6 @@ package org.elasticsearch.action.admin.cluster.shards;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
@@ -31,6 +30,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
@@ -59,7 +59,7 @@ public class TransportClusterSearchShardsAction extends TransportMasterNodeReadA
 
     @Override
     protected ClusterBlockException checkBlock(ClusterSearchShardsRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndices(state, request));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
@@ -70,7 +70,7 @@ public class TransportClusterSearchShardsAction extends TransportMasterNodeReadA
     @Override
     protected void masterOperation(final ClusterSearchShardsRequest request, final ClusterState state, final ActionListener<ClusterSearchShardsResponse> listener) {
         ClusterState clusterState = clusterService.state();
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, request);
+        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request);
         Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(state, request.routing(), request.indices());
         Set<String> nodeIds = new HashSet<>();
         GroupShardsIterator groupShardsIterator = clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, request.preference());
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
index 457b6e69383..2654ac0c269 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
@@ -22,12 +22,12 @@ package org.elasticsearch.action.admin.cluster.snapshots.create;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.snapshots.SnapshotInfo;
@@ -66,7 +66,7 @@ public class TransportCreateSnapshotAction extends TransportMasterNodeAction<Cre
         if (clusterBlockException != null) {
             return clusterBlockException;
         }
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.READ, indexNameExpressionResolver.concreteIndices(state, request));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.READ, indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
index 4e3c777b7d1..423e38cd258 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
@@ -22,12 +22,12 @@ package org.elasticsearch.action.admin.cluster.snapshots.delete;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.snapshots.SnapshotsService;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
index 478146de357..0198102a200 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
@@ -22,12 +22,12 @@ package org.elasticsearch.action.admin.cluster.snapshots.get;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
index ae802342824..cbbc195370c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
@@ -22,12 +22,12 @@ package org.elasticsearch.action.admin.cluster.snapshots.restore;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.snapshots.RestoreInfo;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
index 45c3f89919e..0bc13db9644 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
@@ -28,10 +28,10 @@ import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
index fc19dd986de..efa156eaa0c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
@@ -24,13 +24,13 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
index 72b686ccfde..4bd826237c5 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -32,6 +31,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaData.Custom;
 import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -106,7 +106,7 @@ public class TransportClusterStateAction extends TransportMasterNodeReadAction<C
             }
 
             if (request.indices().length > 0) {
-                String[] indices = indexNameExpressionResolver.concreteIndices(currentState, request);
+                String[] indices = indexNameExpressionResolver.concreteIndexNames(currentState, request);
                 for (String filteredIndex : indices) {
                     IndexMetaData indexMetaData = currentState.metaData().index(filteredIndex);
                     if (indexMetaData != null) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java
index ec7017160c0..f8304bf76a9 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java
@@ -31,7 +31,7 @@ import org.elasticsearch.common.xcontent.XContentBuilderString;
 import org.elasticsearch.index.cache.query.QueryCacheStats;
 import org.elasticsearch.index.engine.SegmentsStats;
 import org.elasticsearch.index.fielddata.FieldDataStats;
-import org.elasticsearch.index.percolator.PercolateStats;
+import org.elasticsearch.index.percolator.PercolatorQueryCacheStats;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.store.StoreStats;
 import org.elasticsearch.search.suggest.completion.CompletionStats;
@@ -48,7 +48,7 @@ public class ClusterStatsIndices implements ToXContent, Streamable {
     private QueryCacheStats queryCache;
     private CompletionStats completion;
     private SegmentsStats segments;
-    private PercolateStats percolate;
+    private PercolatorQueryCacheStats percolatorCache;
 
     private ClusterStatsIndices() {
     }
@@ -62,7 +62,7 @@ public class ClusterStatsIndices implements ToXContent, Streamable {
         this.queryCache = new QueryCacheStats();
         this.completion = new CompletionStats();
         this.segments = new SegmentsStats();
-        this.percolate = new PercolateStats();
+        this.percolatorCache = new PercolatorQueryCacheStats();
 
         for (ClusterStatsNodeResponse r : nodeResponses) {
             for (org.elasticsearch.action.admin.indices.stats.ShardStats shardStats : r.shardsStats()) {
@@ -85,7 +85,7 @@ public class ClusterStatsIndices implements ToXContent, Streamable {
                 queryCache.add(shardCommonStats.queryCache);
                 completion.add(shardCommonStats.completion);
                 segments.add(shardCommonStats.segments);
-                percolate.add(shardCommonStats.percolate);
+                percolatorCache.add(shardCommonStats.percolatorCache);
             }
         }
 
@@ -128,8 +128,8 @@ public class ClusterStatsIndices implements ToXContent, Streamable {
         return segments;
     }
 
-    public PercolateStats getPercolate() {
-        return percolate;
+    public PercolatorQueryCacheStats getPercolatorCache() {
+        return percolatorCache;
     }
 
     @Override
@@ -142,7 +142,7 @@ public class ClusterStatsIndices implements ToXContent, Streamable {
         queryCache = QueryCacheStats.readQueryCacheStats(in);
         completion = CompletionStats.readCompletionStats(in);
         segments = SegmentsStats.readSegmentsStats(in);
-        percolate = PercolateStats.readPercolateStats(in);
+        percolatorCache = PercolatorQueryCacheStats.readPercolateStats(in);
     }
 
     @Override
@@ -155,7 +155,7 @@ public class ClusterStatsIndices implements ToXContent, Streamable {
         queryCache.writeTo(out);
         completion.writeTo(out);
         segments.writeTo(out);
-        percolate.writeTo(out);
+        percolatorCache.writeTo(out);
     }
 
     public static ClusterStatsIndices readIndicesStats(StreamInput in) throws IOException {
@@ -178,7 +178,7 @@ public class ClusterStatsIndices implements ToXContent, Streamable {
         queryCache.toXContent(builder, params);
         completion.toXContent(builder, params);
         segments.toXContent(builder, params);
-        percolate.toXContent(builder, params);
+        percolatorCache.toXContent(builder, params);
         return builder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java
index 5604616ed39..6020aa1a104 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java
@@ -302,7 +302,6 @@ public class ClusterStatsNodes implements ToXContent, Streamable {
 
         int availableProcessors;
         int allocatedProcessors;
-        long availableMemory;
         final ObjectIntHashMap<String> names;
 
         public OsStats() {
@@ -326,15 +325,10 @@ public class ClusterStatsNodes implements ToXContent, Streamable {
             return allocatedProcessors;
         }
 
-        public ByteSizeValue getAvailableMemory() {
-            return new ByteSizeValue(availableMemory);
-        }
-
         @Override
         public void readFrom(StreamInput in) throws IOException {
             availableProcessors = in.readVInt();
             allocatedProcessors = in.readVInt();
-            availableMemory = in.readLong();
             int size = in.readVInt();
             names.clear();
             for (int i = 0; i < size; i++) {
@@ -346,7 +340,6 @@ public class ClusterStatsNodes implements ToXContent, Streamable {
         public void writeTo(StreamOutput out) throws IOException {
             out.writeVInt(availableProcessors);
             out.writeVInt(allocatedProcessors);
-            out.writeLong(availableMemory);
             out.writeVInt(names.size());
             for (ObjectIntCursor<String> name : names) {
                 out.writeString(name.key);
@@ -365,9 +358,6 @@ public class ClusterStatsNodes implements ToXContent, Streamable {
             static final XContentBuilderString ALLOCATED_PROCESSORS = new XContentBuilderString("allocated_processors");
             static final XContentBuilderString NAME = new XContentBuilderString("name");
             static final XContentBuilderString NAMES = new XContentBuilderString("names");
-            static final XContentBuilderString MEM = new XContentBuilderString("mem");
-            static final XContentBuilderString TOTAL = new XContentBuilderString("total");
-            static final XContentBuilderString TOTAL_IN_BYTES = new XContentBuilderString("total_in_bytes");
             static final XContentBuilderString COUNT = new XContentBuilderString("count");
         }
 
@@ -375,10 +365,6 @@ public class ClusterStatsNodes implements ToXContent, Streamable {
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.field(Fields.AVAILABLE_PROCESSORS, availableProcessors);
             builder.field(Fields.ALLOCATED_PROCESSORS, allocatedProcessors);
-            builder.startObject(Fields.MEM);
-            builder.byteSizeField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, availableMemory);
-            builder.endObject();
-
             builder.startArray(Fields.NAMES);
             for (ObjectIntCursor<String> name : names) {
                 builder.startObject();
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index 4087fe6cfce..6d1614eb485 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -28,10 +28,10 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -56,7 +56,7 @@ public class TransportClusterStatsAction extends TransportNodesAction<ClusterSta
 
     private static final CommonStatsFlags SHARD_STATS_FLAGS = new CommonStatsFlags(CommonStatsFlags.Flag.Docs, CommonStatsFlags.Flag.Store,
             CommonStatsFlags.Flag.FieldData, CommonStatsFlags.Flag.QueryCache, CommonStatsFlags.Flag.Completion, CommonStatsFlags.Flag.Segments,
-            CommonStatsFlags.Flag.Percolate);
+            CommonStatsFlags.Flag.PercolatorCache);
 
     private final NodeService nodeService;
     private final IndicesService indicesService;
@@ -99,13 +99,13 @@ public class TransportClusterStatsAction extends TransportNodesAction<ClusterSta
     @Override
     protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest) {
         NodeInfo nodeInfo = nodeService.info(false, true, false, true, false, true, false, true, false);
-        NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE, false, true, true, false, true, false, false, false, false, false);
+        NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE, false, true, true, false, true, false, false, false, false, false, false);
         List<ShardStats> shardsStats = new ArrayList<>();
         for (IndexService indexService : indicesService) {
             for (IndexShard indexShard : indexService) {
                 if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) {
                     // only report on fully started shards
-                    shardsStats.add(new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS), indexShard.commitStats()));
+                    shardsStats.add(new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexService.cache().getPercolatorQueryCache(), indexShard, SHARD_STATS_FLAGS), indexShard.commitStats()));
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java
index 105d596bad8..370b668f659 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java
@@ -22,11 +22,11 @@ package org.elasticsearch.action.admin.cluster.tasks;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.PendingClusterTask;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
index d37053e056b..218b84e68ae 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -31,6 +30,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.AliasAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaDataIndexAliasesService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.action.admin.indices.alias.delete.AliasesNotFoundException;
@@ -90,11 +90,11 @@ public class TransportIndicesAliasesAction extends TransportMasterNodeAction<Ind
         Set<String> aliases = new HashSet<>();
         for (AliasActions action : actions) {
             //expand indices
-            String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request.indicesOptions(), action.indices());
+            String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request.indicesOptions(), action.indices());
             //collect the aliases
             Collections.addAll(aliases, action.aliases());
             for (String index : concreteIndices) {
-                for (String alias : action.concreteAliases(state.metaData(), index)) { 
+                for (String alias : action.concreteAliases(state.metaData(), index)) {
                     AliasAction finalAction = new AliasAction(action.aliasAction());
                     finalAction.index(index);
                     finalAction.alias(alias);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/TransportAliasesExistAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/TransportAliasesExistAction.java
index 5f92587f138..8ca09dbb67e 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/TransportAliasesExistAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/TransportAliasesExistAction.java
@@ -22,11 +22,11 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -50,7 +50,7 @@ public class TransportAliasesExistAction extends TransportMasterNodeReadAction<G
 
     @Override
     protected ClusterBlockException checkBlock(GetAliasesRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndices(state, request));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
@@ -60,7 +60,7 @@ public class TransportAliasesExistAction extends TransportMasterNodeReadAction<G
 
     @Override
     protected void masterOperation(GetAliasesRequest request, ClusterState state, ActionListener<AliasesExistResponse> listener) {
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
+        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request);
         boolean result = state.metaData().hasAliases(request.aliases(), concreteIndices);
         listener.onResponse(new AliasesExistResponse(result));
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java
index 9c2c2f03b57..061f916c2e0 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java
@@ -21,12 +21,12 @@ package org.elasticsearch.action.admin.indices.alias.get;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -53,7 +53,7 @@ public class TransportGetAliasesAction extends TransportMasterNodeReadAction<Get
 
     @Override
     protected ClusterBlockException checkBlock(GetAliasesRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndices(state, request));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
@@ -63,7 +63,7 @@ public class TransportGetAliasesAction extends TransportMasterNodeReadAction<Get
 
     @Override
     protected void masterOperation(GetAliasesRequest request, ClusterState state, ActionListener<GetAliasesResponse> listener) {
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
+        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request);
         @SuppressWarnings("unchecked")
                 ImmutableOpenMap<String, List<AliasMetaData>> result = (ImmutableOpenMap) state.metaData().findAliases(request.aliases(), concreteIndices);
         listener.onResponse(new GetAliasesResponse(result));
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
index 0541ac31505..0edae5eb1bc 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
@@ -32,11 +32,11 @@ import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardsIterator;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.FastStringReader;
 import org.elasticsearch.common.settings.Settings;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java
index bc229d72b1b..59cd95044cc 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java
@@ -22,13 +22,13 @@ package org.elasticsearch.action.admin.indices.cache.clear;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
@@ -77,7 +77,7 @@ public class TransportClearIndicesCacheAction extends TransportBroadcastByNodeAc
 
     @Override
     protected EmptyResult shardOperation(ClearIndicesCacheRequest request, ShardRouting shardRouting) {
-        IndexService service = indicesService.indexService(shardRouting.getIndexName());
+        IndexService service = indicesService.indexService(shardRouting.index());
         if (service != null) {
             IndexShard shard = service.getShardOrNull(shardRouting.id());
             boolean clearedAtLeastOne = false;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java
index f8bbebf7db8..4fbfc7e72ab 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java
@@ -23,17 +23,19 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -46,7 +48,8 @@ public class TransportCloseIndexAction extends TransportMasterNodeAction<CloseIn
     private final MetaDataIndexStateService indexStateService;
     private final DestructiveOperations destructiveOperations;
     private volatile boolean closeIndexEnabled;
-    public static final Setting<Boolean> CLUSTER_INDICES_CLOSE_ENABLE_SETTING = Setting.boolSetting("cluster.indices.close.enable", true, true, Setting.Scope.CLUSTER);
+    public static final Setting<Boolean> CLUSTER_INDICES_CLOSE_ENABLE_SETTING =
+        Setting.boolSetting("cluster.indices.close.enable", true, Property.Dynamic, Property.NodeScope);
 
     @Inject
     public TransportCloseIndexAction(Settings settings, TransportService transportService, ClusterService clusterService,
@@ -86,12 +89,12 @@ public class TransportCloseIndexAction extends TransportMasterNodeAction<CloseIn
 
     @Override
     protected ClusterBlockException checkBlock(CloseIndexRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indexNameExpressionResolver.concreteIndices(state, request));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
     protected void masterOperation(final CloseIndexRequest request, final ClusterState state, final ActionListener<CloseIndexResponse> listener) {
-        final String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
+        final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
         CloseIndexClusterStateUpdateRequest updateRequest = new CloseIndexClusterStateUpdateRequest()
                 .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout())
                 .indices(concreteIndices);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java
index 98a002cc2fb..7b47a46a236 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java
@@ -22,13 +22,13 @@ package org.elasticsearch.action.admin.indices.create;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
index 28bf46f798f..489001d9b89 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
@@ -23,18 +23,23 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaDataDeleteIndexService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
 /**
  * Delete index action.
  */
@@ -70,13 +75,13 @@ public class TransportDeleteIndexAction extends TransportMasterNodeAction<Delete
 
     @Override
     protected ClusterBlockException checkBlock(DeleteIndexRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indexNameExpressionResolver.concreteIndices(state, request));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
     protected void masterOperation(final DeleteIndexRequest request, final ClusterState state, final ActionListener<DeleteIndexResponse> listener) {
-        final String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
-        if (concreteIndices.length == 0) {
+        final Set<Index> concreteIndices = new HashSet<>(Arrays.asList(indexNameExpressionResolver.concreteIndices(state, request)));
+        if (concreteIndices.isEmpty()) {
             listener.onResponse(new DeleteIndexResponse(true));
             return;
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java
index acda370d7ff..c451e50b77c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java
@@ -23,11 +23,11 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -60,7 +60,7 @@ public class TransportIndicesExistsAction extends TransportMasterNodeReadAction<
     protected ClusterBlockException checkBlock(IndicesExistsRequest request, ClusterState state) {
         //make sure through indices options that the concrete indices call never throws IndexMissingException
         IndicesOptions indicesOptions = IndicesOptions.fromOptions(true, true, request.indicesOptions().expandWildcardsOpen(), request.indicesOptions().expandWildcardsClosed());
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndices(state, indicesOptions, request.indices()));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndexNames(state, indicesOptions, request.indices()));
     }
 
     @Override
@@ -68,7 +68,7 @@ public class TransportIndicesExistsAction extends TransportMasterNodeReadAction<
         boolean exists;
         try {
             // Similar as the previous behaviour, but now also aliases and wildcards are supported.
-            indexNameExpressionResolver.concreteIndices(state, request);
+            indexNameExpressionResolver.concreteIndexNames(state, request);
             exists = true;
         } catch (IndexNotFoundException e) {
             exists = false;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TransportTypesExistsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TransportTypesExistsAction.java
index 2fd92451752..e1cf5be1aca 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TransportTypesExistsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TransportTypesExistsAction.java
@@ -21,12 +21,12 @@ package org.elasticsearch.action.admin.indices.exists.types;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -57,12 +57,12 @@ public class TransportTypesExistsAction extends TransportMasterNodeReadAction<Ty
 
     @Override
     protected ClusterBlockException checkBlock(TypesExistsRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndices(state, request));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
     protected void masterOperation(final TypesExistsRequest request, final ClusterState state, final ActionListener<TypesExistsResponse> listener) {
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request.indicesOptions(), request.indices());
+        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request.indicesOptions(), request.indices());
         if (concreteIndices.length == 0) {
             listener.onResponse(new TypesExistsResponse(false));
             return;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java
index d2a8f1abcbf..8bb124d8fc4 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java
@@ -23,8 +23,8 @@ import org.elasticsearch.action.ReplicationResponse;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.replication.TransportBroadcastReplicationAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.ShardId;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
index 302bdafc471..3c22209813f 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
@@ -22,12 +22,11 @@ package org.elasticsearch.action.admin.indices.flush;
 import org.elasticsearch.action.ReplicationResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
-import org.elasticsearch.cluster.ClusterService;
-import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -46,10 +45,9 @@ public class TransportShardFlushAction extends TransportReplicationAction<ShardF
     @Inject
     public TransportShardFlushAction(Settings settings, TransportService transportService, ClusterService clusterService,
                                      IndicesService indicesService, ThreadPool threadPool, ShardStateAction shardStateAction,
-                                     MappingUpdatedAction mappingUpdatedAction, ActionFilters actionFilters,
-                                     IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, NAME, transportService, clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedAction,
-                actionFilters, indexNameExpressionResolver, ShardFlushRequest::new, ShardFlushRequest::new, ThreadPool.Names.FLUSH);
+                                     ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(settings, NAME, transportService, clusterService, indicesService, threadPool, shardStateAction,
+            actionFilters, indexNameExpressionResolver, ShardFlushRequest::new, ShardFlushRequest::new, ThreadPool.Names.FLUSH);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java
index 0119b1693aa..18ac88e1b30 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java
@@ -22,13 +22,13 @@ package org.elasticsearch.action.admin.indices.forcemerge;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java
index 1b9180ce192..097444bcb68 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest.Feature;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.info.TransportClusterInfoAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
@@ -32,6 +31,7 @@ import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
@@ -60,7 +60,7 @@ public class TransportGetIndexAction extends TransportClusterInfoAction<GetIndex
 
     @Override
     protected ClusterBlockException checkBlock(GetIndexRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndices(state, request));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java
index 82d6c21f022..d9031807ae8 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java
@@ -22,9 +22,9 @@ package org.elasticsearch.action.admin.indices.mapping.get;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -56,7 +56,7 @@ public class TransportGetFieldMappingsAction extends HandledTransportAction<GetF
     @Override
     protected void doExecute(GetFieldMappingsRequest request, final ActionListener<GetFieldMappingsResponse> listener) {
         ClusterState clusterState = clusterService.state();
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, request);
+        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request);
         final AtomicInteger indexCounter = new AtomicInteger();
         final AtomicInteger completionCounter = new AtomicInteger(concreteIndices.length);
         final AtomicReferenceArray<Object> indexResponses = new AtomicReferenceArray<>(concreteIndices.length);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
index 3d11df97dee..e886af25fbb 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
@@ -23,12 +23,12 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardsIterator;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java
index bff9b4e2ab6..293f5a0e677 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java
@@ -22,12 +22,12 @@ package org.elasticsearch.action.admin.indices.mapping.get;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.info.TransportClusterInfoAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -52,7 +52,7 @@ public class TransportGetMappingsAction extends TransportClusterInfoAction<GetMa
 
     @Override
     protected ClusterBlockException checkBlock(GetMappingsRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndices(state, request));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
index 3a1aadd7650..c638a429b10 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
@@ -32,9 +32,12 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.Index;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
@@ -65,6 +68,7 @@ public class PutMappingRequest extends AcknowledgedRequest<PutMappingRequest> im
     private String source;
 
     private boolean updateAllTypes = false;
+    private Index concreteIndex;
 
     public PutMappingRequest() {
     }
@@ -90,6 +94,10 @@ public class PutMappingRequest extends AcknowledgedRequest<PutMappingRequest> im
         } else if (source.isEmpty()) {
             validationException = addValidationError("mapping source is empty", validationException);
         }
+        if (concreteIndex != null && (indices != null && indices.length > 0)) {
+            validationException = addValidationError("either concrete index or unresolved indices can be set, concrete index: ["
+                + concreteIndex + "] and indices: " + Arrays.asList(indices) , validationException);
+        }
         return validationException;
     }
 
@@ -102,6 +110,22 @@ public class PutMappingRequest extends AcknowledgedRequest<PutMappingRequest> im
         return this;
     }
 
+    /**
+     * Sets a concrete index for this put mapping request.
+     */
+    public PutMappingRequest setConcreteIndex(Index index) {
+        Objects.requireNonNull(indices, "index must not be null");
+        this.concreteIndex = index;
+        return this;
+    }
+
+    /**
+     * Returns a concrete index for this mapping or <code>null</code> if no concrete index is defined
+     */
+    public Index getConcreteIndex() {
+        return concreteIndex;
+    }
+
     /**
      * The indices the mappings will be put.
      */
@@ -259,6 +283,7 @@ public class PutMappingRequest extends AcknowledgedRequest<PutMappingRequest> im
         source = in.readString();
         updateAllTypes = in.readBoolean();
         readTimeout(in);
+        concreteIndex = in.readOptionalWriteable(Index::new);
     }
 
     @Override
@@ -270,5 +295,6 @@ public class PutMappingRequest extends AcknowledgedRequest<PutMappingRequest> im
         out.writeString(source);
         out.writeBoolean(updateAllTypes);
         writeTimeout(out);
+        out.writeOptionalWriteable(concreteIndex);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java
index 28f289b86c6..c21c40cf041 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.Index;
 
 import java.util.Map;
 
@@ -40,6 +41,11 @@ public class PutMappingRequestBuilder extends AcknowledgedRequestBuilder<PutMapp
         return this;
     }
 
+    public PutMappingRequestBuilder setConcreteIndex(Index index) {
+        request.setConcreteIndex(index);
+        return this;
+    }
+
     /**
      * Specifies what type of requested indices to ignore and wildcard indices expressions.
      * <p>
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
index b82c5d3a626..46535350154 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
@@ -22,15 +22,16 @@ package org.elasticsearch.action.admin.indices.mapping.put;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaDataMappingService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -63,13 +64,19 @@ public class TransportPutMappingAction extends TransportMasterNodeAction<PutMapp
 
     @Override
     protected ClusterBlockException checkBlock(PutMappingRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indexNameExpressionResolver.concreteIndices(state, request));
+        String[] indices;
+        if (request.getConcreteIndex() == null) {
+            indices = indexNameExpressionResolver.concreteIndexNames(state, request);
+        } else {
+            indices = new String[] {request.getConcreteIndex().getName()};
+        }
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indices);
     }
 
     @Override
     protected void masterOperation(final PutMappingRequest request, final ClusterState state, final ActionListener<PutMappingResponse> listener) {
         try {
-            final String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
+            final Index[] concreteIndices = request.getConcreteIndex() == null ? indexNameExpressionResolver.concreteIndices(state, request) : new Index[] {request.getConcreteIndex()};
             PutMappingClusterStateUpdateRequest updateRequest = new PutMappingClusterStateUpdateRequest()
                     .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout())
                     .indices(concreteIndices).type(request.type())
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java
index 7ffb30b9534..50e79036694 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java
@@ -23,15 +23,16 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -73,12 +74,12 @@ public class TransportOpenIndexAction extends TransportMasterNodeAction<OpenInde
 
     @Override
     protected ClusterBlockException checkBlock(OpenIndexRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indexNameExpressionResolver.concreteIndices(state, request));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
     protected void masterOperation(final OpenIndexRequest request, final ClusterState state, final ActionListener<OpenIndexResponse> listener) {
-        final String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
+        final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
         OpenIndexClusterStateUpdateRequest updateRequest = new OpenIndexClusterStateUpdateRequest()
                 .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout())
                 .indices(concreteIndices);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java
index 8590fc210a0..01f37527374 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java
@@ -22,13 +22,13 @@ package org.elasticsearch.action.admin.indices.recovery;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java
index bd879e0eaa9..34bf39daabd 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java
@@ -24,8 +24,8 @@ import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.replication.BasicReplicationRequest;
 import org.elasticsearch.action.support.replication.TransportBroadcastReplicationAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.ShardId;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
index 2dd41f7801d..e3155614337 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
@@ -23,12 +23,11 @@ import org.elasticsearch.action.ReplicationResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.replication.BasicReplicationRequest;
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
-import org.elasticsearch.cluster.ClusterService;
-import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -48,10 +47,9 @@ public class TransportShardRefreshAction extends TransportReplicationAction<Basi
     @Inject
     public TransportShardRefreshAction(Settings settings, TransportService transportService, ClusterService clusterService,
                                        IndicesService indicesService, ThreadPool threadPool, ShardStateAction shardStateAction,
-                                       MappingUpdatedAction mappingUpdatedAction, ActionFilters actionFilters,
-                                       IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, NAME, transportService, clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedAction,
-                actionFilters, indexNameExpressionResolver, BasicReplicationRequest::new, BasicReplicationRequest::new, ThreadPool.Names.REFRESH);
+                                       ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(settings, NAME, transportService, clusterService, indicesService, threadPool, shardStateAction,
+            actionFilters, indexNameExpressionResolver, BasicReplicationRequest::new, BasicReplicationRequest::new, ThreadPool.Names.REFRESH);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java
index fd45e22a171..8df46719c7b 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java
@@ -22,13 +22,13 @@ package org.elasticsearch.action.admin.indices.segments;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
@@ -93,7 +93,7 @@ public class TransportIndicesSegmentsAction extends TransportBroadcastByNodeActi
 
     @Override
     protected ShardSegments shardOperation(IndicesSegmentsRequest request, ShardRouting shardRouting) {
-        IndexService indexService = indicesService.indexServiceSafe(shardRouting.getIndexName());
+        IndexService indexService = indicesService.indexServiceSafe(shardRouting.index());
         IndexShard indexShard = indexService.getShard(shardRouting.id());
         return new ShardSegments(indexShard.routingEntry(), indexShard.segments(request.verbose()));
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java
index b9393d7fdd7..f09d3fb559c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java
@@ -22,18 +22,19 @@ package org.elasticsearch.action.admin.indices.settings.get;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -61,7 +62,7 @@ public class TransportGetSettingsAction extends TransportMasterNodeReadAction<Ge
 
     @Override
     protected ClusterBlockException checkBlock(GetSettingsRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndices(state, request));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
 
@@ -72,9 +73,9 @@ public class TransportGetSettingsAction extends TransportMasterNodeReadAction<Ge
 
     @Override
     protected void masterOperation(GetSettingsRequest request, ClusterState state, ActionListener<GetSettingsResponse> listener) {
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
+        Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
         ImmutableOpenMap.Builder<String, Settings> indexToSettingsBuilder = ImmutableOpenMap.builder();
-        for (String concreteIndex : concreteIndices) {
+        for (Index concreteIndex : concreteIndices) {
             IndexMetaData indexMetaData = state.getMetaData().index(concreteIndex);
             if (indexMetaData == null) {
                 continue;
@@ -93,7 +94,7 @@ public class TransportGetSettingsAction extends TransportMasterNodeReadAction<Ge
                 }
                 settings = settingsBuilder.build();
             }
-            indexToSettingsBuilder.put(concreteIndex, settings);
+            indexToSettingsBuilder.put(concreteIndex.getName(), settings);
         }
         listener.onResponse(new GetSettingsResponse(indexToSettingsBuilder.build()));
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
index 415532288cd..b20f636977a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
@@ -22,7 +22,6 @@ package org.elasticsearch.action.admin.indices.settings.put;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -30,8 +29,10 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaDataUpdateSettingsService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -65,7 +66,7 @@ public class TransportUpdateSettingsAction extends TransportMasterNodeAction<Upd
         if (request.settings().getAsMap().size() == 1 && IndexMetaData.INDEX_BLOCKS_METADATA_SETTING.exists(request.settings()) || IndexMetaData.INDEX_READ_ONLY_SETTING.exists(request.settings())) {
             return null;
         }
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indexNameExpressionResolver.concreteIndices(state, request));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
@@ -75,10 +76,11 @@ public class TransportUpdateSettingsAction extends TransportMasterNodeAction<Upd
 
     @Override
     protected void masterOperation(final UpdateSettingsRequest request, final ClusterState state, final ActionListener<UpdateSettingsResponse> listener) {
-        final String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
+        final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
         UpdateSettingsClusterStateUpdateRequest clusterStateUpdateRequest = new UpdateSettingsClusterStateUpdateRequest()
                 .indices(concreteIndices)
                 .settings(request.settings())
+                .setPreserveExisting(request.isPreserveExisting())
                 .ackTimeout(request.timeout())
                 .masterNodeTimeout(request.masterNodeTimeout());
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsClusterStateUpdateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsClusterStateUpdateRequest.java
index ac6f6f3d496..6c21768dbb7 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsClusterStateUpdateRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsClusterStateUpdateRequest.java
@@ -29,8 +29,23 @@ public class UpdateSettingsClusterStateUpdateRequest extends IndicesClusterState
 
     private Settings settings;
 
-    public UpdateSettingsClusterStateUpdateRequest() {
+    private boolean preserveExisting = false;
 
+    /**
+     * Returns <code>true</code> iff the settings update should only add but not update settings. If the setting already exists
+     * it should not be overwritten by this update. The default is <code>false</code>
+     */
+    public boolean isPreserveExisting() {
+        return preserveExisting;
+    }
+
+    /**
+     * Iff set to <code>true</code> this settings update will only add settings not already set on an index. Existing settings remain
+     * unchanged.
+     */
+    public UpdateSettingsClusterStateUpdateRequest setPreserveExisting(boolean preserveExisting) {
+        this.preserveExisting = preserveExisting;
+        return this;
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
index c654d6926fa..fb4525a6842 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
@@ -47,6 +47,7 @@ public class UpdateSettingsRequest extends AcknowledgedRequest<UpdateSettingsReq
     private String[] indices;
     private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, true, true);
     private Settings settings = EMPTY_SETTINGS;
+    private boolean preserveExisting = false;
 
     public UpdateSettingsRequest() {
     }
@@ -127,6 +128,23 @@ public class UpdateSettingsRequest extends AcknowledgedRequest<UpdateSettingsReq
         return this;
     }
 
+    /**
+     * Returns <code>true</code> iff the settings update should only add but not update settings. If the setting already exists
+     * it should not be overwritten by this update. The default is <code>false</code>
+     */
+    public boolean isPreserveExisting() {
+        return preserveExisting;
+    }
+
+    /**
+     * Iff set to <code>true</code> this settings update will only add settings not already set on an index. Existing settings remain
+     * unchanged.
+     */
+    public UpdateSettingsRequest setPreserveExisting(boolean preserveExisting) {
+        this.preserveExisting = preserveExisting;
+        return this;
+    }
+
     /**
      * Sets the settings to be updated (either json/yaml/properties format)
      */
@@ -149,6 +167,7 @@ public class UpdateSettingsRequest extends AcknowledgedRequest<UpdateSettingsReq
         indicesOptions = IndicesOptions.readIndicesOptions(in);
         settings = readSettingsFromStream(in);
         readTimeout(in);
+        preserveExisting = in.readBoolean();
     }
 
     @Override
@@ -158,5 +177,6 @@ public class UpdateSettingsRequest extends AcknowledgedRequest<UpdateSettingsReq
         indicesOptions.writeIndicesOptions(out);
         writeSettingsToStream(settings, out);
         writeTimeout(out);
+        out.writeBoolean(preserveExisting);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java
index 770c54b1a3e..36dfbf3b2d4 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java
@@ -84,4 +84,9 @@ public class UpdateSettingsRequestBuilder extends AcknowledgedRequestBuilder<Upd
         request.settings(source);
         return this;
     }
+
+    public UpdateSettingsRequestBuilder setPreserveExisting(boolean preserveExisting) {
+        request.setPreserveExisting(preserveExisting);
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
index 5e22bc89144..bf4dbd3359e 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
@@ -38,6 +37,7 @@ import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenIntMap;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
@@ -87,7 +87,7 @@ public class TransportIndicesShardStoresAction extends TransportMasterNodeReadAc
     protected void masterOperation(IndicesShardStoresRequest request, ClusterState state, ActionListener<IndicesShardStoresResponse> listener) {
         final RoutingTable routingTables = state.routingTable();
         final RoutingNodes routingNodes = state.getRoutingNodes();
-        final String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
+        final String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request);
         final Set<ShardId> shardIdsToFetch = new HashSet<>();
 
         logger.trace("using cluster state version [{}] to determine shards", state.version());
@@ -115,7 +115,7 @@ public class TransportIndicesShardStoresAction extends TransportMasterNodeReadAc
 
     @Override
     protected ClusterBlockException checkBlock(IndicesShardStoresRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndices(state, request));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     private class AsyncShardStoresInfoFetches {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
index 676eec104b1..b130e6b378f 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
@@ -32,16 +32,16 @@ import org.elasticsearch.index.engine.SegmentsStats;
 import org.elasticsearch.index.fielddata.FieldDataStats;
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.get.GetStats;
+import org.elasticsearch.index.percolator.PercolatorQueryCache;
 import org.elasticsearch.index.shard.IndexingStats;
 import org.elasticsearch.index.merge.MergeStats;
-import org.elasticsearch.index.percolator.PercolateStats;
+import org.elasticsearch.index.percolator.PercolatorQueryCacheStats;
 import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.refresh.RefreshStats;
 import org.elasticsearch.index.search.stats.SearchStats;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.store.StoreStats;
-import org.elasticsearch.index.suggest.stats.SuggestStats;
 import org.elasticsearch.index.translog.TranslogStats;
 import org.elasticsearch.index.warmer.WarmerStats;
 import org.elasticsearch.indices.IndicesQueryCache;
@@ -101,14 +101,14 @@ public class CommonStats implements Streamable, ToXContent {
                 case Segments:
                     segments = new SegmentsStats();
                     break;
-                case Percolate:
-                    percolate = new PercolateStats();
+                case PercolatorCache:
+                    percolatorCache = new PercolatorQueryCacheStats();
                     break;
                 case Translog:
                     translog = new TranslogStats();
                     break;
                 case Suggest:
-                    suggest = new SuggestStats();
+                    // skip
                     break;
                 case RequestCache:
                     requestCache = new RequestCacheStats();
@@ -123,7 +123,8 @@ public class CommonStats implements Streamable, ToXContent {
     }
 
 
-    public CommonStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, CommonStatsFlags flags) {
+    public CommonStats(IndicesQueryCache indicesQueryCache, PercolatorQueryCache percolatorQueryCache,
+                       IndexShard indexShard, CommonStatsFlags flags) {
 
         CommonStatsFlags.Flag[] setFlags = flags.getFlags();
 
@@ -168,14 +169,14 @@ public class CommonStats implements Streamable, ToXContent {
                 case Segments:
                     segments = indexShard.segmentStats(flags.includeSegmentFileSizes());
                     break;
-                case Percolate:
-                    percolate = indexShard.percolateStats();
+                case PercolatorCache:
+                    percolatorCache = percolatorQueryCache.getStats(indexShard.shardId());
                     break;
                 case Translog:
                     translog = indexShard.translogStats();
                     break;
                 case Suggest:
-                    suggest = indexShard.suggestStats();
+                    // skip
                     break;
                 case RequestCache:
                     requestCache = indexShard.requestCache().stats();
@@ -223,7 +224,7 @@ public class CommonStats implements Streamable, ToXContent {
     public FieldDataStats fieldData;
 
     @Nullable
-    public PercolateStats percolate;
+    public PercolatorQueryCacheStats percolatorCache;
 
     @Nullable
     public CompletionStats completion;
@@ -234,9 +235,6 @@ public class CommonStats implements Streamable, ToXContent {
     @Nullable
     public TranslogStats translog;
 
-    @Nullable
-    public SuggestStats suggest;
-
     @Nullable
     public RequestCacheStats requestCache;
 
@@ -333,13 +331,13 @@ public class CommonStats implements Streamable, ToXContent {
         } else {
             fieldData.add(stats.getFieldData());
         }
-        if (percolate == null) {
-            if (stats.getPercolate() != null) {
-                percolate = new PercolateStats();
-                percolate.add(stats.getPercolate());
+        if (percolatorCache == null) {
+            if (stats.getPercolatorCache() != null) {
+                percolatorCache = new PercolatorQueryCacheStats();
+                percolatorCache.add(stats.getPercolatorCache());
             }
         } else {
-            percolate.add(stats.getPercolate());
+            percolatorCache.add(stats.getPercolatorCache());
         }
         if (completion == null) {
             if (stats.getCompletion() != null) {
@@ -365,14 +363,6 @@ public class CommonStats implements Streamable, ToXContent {
         } else {
             translog.add(stats.getTranslog());
         }
-        if (suggest == null) {
-            if (stats.getSuggest() != null) {
-                suggest = new SuggestStats();
-                suggest.add(stats.getSuggest());
-            }
-        } else {
-            suggest.add(stats.getSuggest());
-        }
         if (requestCache == null) {
             if (stats.getRequestCache() != null) {
                 requestCache = new RequestCacheStats();
@@ -447,8 +437,8 @@ public class CommonStats implements Streamable, ToXContent {
     }
 
     @Nullable
-    public PercolateStats getPercolate() {
-        return percolate;
+    public PercolatorQueryCacheStats getPercolatorCache() {
+        return percolatorCache;
     }
 
     @Nullable
@@ -466,11 +456,6 @@ public class CommonStats implements Streamable, ToXContent {
         return translog;
     }
 
-    @Nullable
-    public SuggestStats getSuggest() {
-        return suggest;
-    }
-
     @Nullable
     public RequestCacheStats getRequestCache() {
         return requestCache;
@@ -489,7 +474,7 @@ public class CommonStats implements Streamable, ToXContent {
 
     /**
      * Utility method which computes total memory by adding
-     * FieldData, Percolate, Segments (memory, index writer, version map)
+     * FieldData, PercolatorCache, Segments (memory, index writer, version map)
      */
     public ByteSizeValue getTotalMemory() {
         long size = 0;
@@ -499,9 +484,6 @@ public class CommonStats implements Streamable, ToXContent {
         if (this.getQueryCache() != null) {
             size += this.getQueryCache().getMemorySizeInBytes();
         }
-        if (this.getPercolate() != null) {
-            size += this.getPercolate().getMemorySizeInBytes();
-        }
         if (this.getSegments() != null) {
             size += this.getSegments().getMemoryInBytes() +
                     this.getSegments().getIndexWriterMemoryInBytes() +
@@ -547,7 +529,7 @@ public class CommonStats implements Streamable, ToXContent {
             fieldData = FieldDataStats.readFieldDataStats(in);
         }
         if (in.readBoolean()) {
-            percolate = PercolateStats.readPercolateStats(in);
+            percolatorCache = PercolatorQueryCacheStats.readPercolateStats(in);
         }
         if (in.readBoolean()) {
             completion = CompletionStats.readCompletionStats(in);
@@ -556,7 +538,6 @@ public class CommonStats implements Streamable, ToXContent {
             segments = SegmentsStats.readSegmentsStats(in);
         }
         translog = in.readOptionalStreamable(TranslogStats::new);
-        suggest = in.readOptionalStreamable(SuggestStats::new);
         requestCache = in.readOptionalStreamable(RequestCacheStats::new);
         recoveryStats = in.readOptionalStreamable(RecoveryStats::new);
     }
@@ -629,11 +610,11 @@ public class CommonStats implements Streamable, ToXContent {
             out.writeBoolean(true);
             fieldData.writeTo(out);
         }
-        if (percolate == null) {
+        if (percolatorCache == null) {
             out.writeBoolean(false);
         } else {
             out.writeBoolean(true);
-            percolate.writeTo(out);
+            percolatorCache.writeTo(out);
         }
         if (completion == null) {
             out.writeBoolean(false);
@@ -648,7 +629,6 @@ public class CommonStats implements Streamable, ToXContent {
             segments.writeTo(out);
         }
         out.writeOptionalStreamable(translog);
-        out.writeOptionalStreamable(suggest);
         out.writeOptionalStreamable(requestCache);
         out.writeOptionalStreamable(recoveryStats);
     }
@@ -689,8 +669,8 @@ public class CommonStats implements Streamable, ToXContent {
         if (fieldData != null) {
             fieldData.toXContent(builder, params);
         }
-        if (percolate != null) {
-            percolate.toXContent(builder, params);
+        if (percolatorCache != null) {
+            percolatorCache.toXContent(builder, params);
         }
         if (completion != null) {
             completion.toXContent(builder, params);
@@ -701,9 +681,6 @@ public class CommonStats implements Streamable, ToXContent {
         if (translog != null) {
             translog.toXContent(builder, params);
         }
-        if (suggest != null) {
-            suggest.toXContent(builder, params);
-        }
         if (requestCache != null) {
             requestCache.toXContent(builder, params);
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java
index 39608c72a15..e2f250dd577 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java
@@ -190,7 +190,7 @@ public class CommonStatsFlags implements Streamable, Cloneable {
         out.writeStringArrayNullable(groups);
         out.writeStringArrayNullable(fieldDataFields);
         out.writeStringArrayNullable(completionDataFields);
-        if (out.getVersion().onOrAfter(Version.V_5_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) {
             out.writeBoolean(includeSegmentFileSizes);
         }
     }
@@ -208,7 +208,7 @@ public class CommonStatsFlags implements Streamable, Cloneable {
         groups = in.readStringArray();
         fieldDataFields = in.readStringArray();
         completionDataFields = in.readStringArray();
-        if (in.getVersion().onOrAfter(Version.V_5_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) {
             includeSegmentFileSizes = in.readBoolean();
         } else {
             includeSegmentFileSizes = false;
@@ -240,11 +240,11 @@ public class CommonStatsFlags implements Streamable, Cloneable {
         FieldData("fielddata"),
         Docs("docs"),
         Warmer("warmer"),
-        Percolate("percolate"),
+        PercolatorCache("percolator_cache"),
         Completion("completion"),
         Segments("segments"),
         Translog("translog"),
-        Suggest("suggest"),
+        Suggest("suggest"), // unused
         RequestCache("request_cache"),
         Recovery("recovery");
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java
index 3a74d896fba..4a2d137593e 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java
@@ -185,12 +185,12 @@ public class IndicesStatsRequest extends BroadcastRequest<IndicesStatsRequest> {
     }
 
     public IndicesStatsRequest percolate(boolean percolate) {
-        flags.set(Flag.Percolate, percolate);
+        flags.set(Flag.PercolatorCache, percolate);
         return this;
     }
 
     public boolean percolate() {
-        return flags.isSet(Flag.Percolate);
+        return flags.isSet(Flag.PercolatorCache);
     }
 
     public IndicesStatsRequest segments(boolean segments) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
index 0ae21a3ac09..cad919cbd18 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
@@ -152,11 +152,6 @@ public class IndicesStatsRequestBuilder extends BroadcastOperationRequestBuilder
         return this;
     }
 
-    public IndicesStatsRequestBuilder setSuggest(boolean suggest) {
-        request.suggest(suggest);
-        return this;
-    }
-
     public IndicesStatsRequestBuilder setRequestCache(boolean requestCache) {
         request.requestCache(requestCache);
         return this;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
index 2189973d9b7..8c12dfa9fda 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
@@ -22,13 +22,13 @@ package org.elasticsearch.action.admin.indices.stats;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
@@ -140,7 +140,7 @@ public class TransportIndicesStatsAction extends TransportBroadcastByNodeAction<
             flags.fieldDataFields(request.fieldDataFields());
         }
         if (request.percolate()) {
-            flags.set(CommonStatsFlags.Flag.Percolate);
+            flags.set(CommonStatsFlags.Flag.PercolatorCache);
         }
         if (request.segments()) {
             flags.set(CommonStatsFlags.Flag.Segments);
@@ -163,6 +163,6 @@ public class TransportIndicesStatsAction extends TransportBroadcastByNodeAction<
             flags.set(CommonStatsFlags.Flag.Recovery);
         }
 
-        return new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexShard, flags), indexShard.commitStats());
+        return new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexService.cache().getPercolatorQueryCache(), indexShard, flags), indexShard.commitStats());
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java
index 9eab0f80e50..0763f232711 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java
@@ -21,12 +21,12 @@ package org.elasticsearch.action.admin.indices.template.delete;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java
index a43397e48dc..672ca1a9080 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java
@@ -22,12 +22,12 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
index c5fed57d013..02aad2f7ff4 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
@@ -21,13 +21,13 @@ package org.elasticsearch.action.admin.indices.template.put;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/TransportUpgradeStatusAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/TransportUpgradeStatusAction.java
index 6b37f56ed4a..cf288e0cc6f 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/TransportUpgradeStatusAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/TransportUpgradeStatusAction.java
@@ -23,13 +23,13 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java
index f3cf2da9fdd..cdf6f585e53 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java
@@ -25,7 +25,6 @@ import org.elasticsearch.action.PrimaryMissingActionException;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
@@ -35,6 +34,7 @@ import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java
index 527adeaa3e5..403456cb903 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java
@@ -22,13 +22,13 @@ package org.elasticsearch.action.admin.indices.upgrade.post;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaDataUpdateSettingsService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java
index fe02a1541a0..320f0696605 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java
@@ -29,13 +29,13 @@ import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.TransportBroadcastAction;
 import org.elasticsearch.cache.recycler.PageCacheRecycler;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.inject.Inject;
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
index 707bf8de57f..9d9b36ba072 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -37,16 +37,17 @@ import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.update.TransportUpdateAction;
 import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
@@ -245,17 +246,18 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
             if (addFailureIfIndexIsUnavailable(documentRequest, bulkRequest, responses, i, concreteIndices, metaData)) {
                 continue;
             }
-            String concreteIndex = concreteIndices.resolveIfAbsent(documentRequest);
+            Index concreteIndex = concreteIndices.resolveIfAbsent(documentRequest);
             if (request instanceof IndexRequest) {
                 IndexRequest indexRequest = (IndexRequest) request;
                 MappingMetaData mappingMd = null;
-                if (metaData.hasIndex(concreteIndex)) {
-                    mappingMd = metaData.index(concreteIndex).mappingOrDefault(indexRequest.type());
+                final IndexMetaData indexMetaData = metaData.index(concreteIndex);
+                if (indexMetaData != null) {
+                    mappingMd = indexMetaData.mappingOrDefault(indexRequest.type());
                 }
                 try {
-                    indexRequest.process(metaData, mappingMd, allowIdGeneration, concreteIndex);
+                    indexRequest.process(metaData, mappingMd, allowIdGeneration, concreteIndex.getName());
                 } catch (ElasticsearchParseException | RoutingMissingException e) {
-                    BulkItemResponse.Failure failure = new BulkItemResponse.Failure(concreteIndex, indexRequest.type(), indexRequest.id(), e);
+                    BulkItemResponse.Failure failure = new BulkItemResponse.Failure(concreteIndex.getName(), indexRequest.type(), indexRequest.id(), e);
                     BulkItemResponse bulkItemResponse = new BulkItemResponse(i, "index", failure);
                     responses.set(i, bulkItemResponse);
                     // make sure the request gets never processed again
@@ -263,9 +265,9 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
                 }
             } else if (request instanceof DeleteRequest) {
                 try {
-                    TransportDeleteAction.resolveAndValidateRouting(metaData, concreteIndex, (DeleteRequest)request);
+                    TransportDeleteAction.resolveAndValidateRouting(metaData, concreteIndex.getName(), (DeleteRequest)request);
                 } catch(RoutingMissingException e) {
-                    BulkItemResponse.Failure failure = new BulkItemResponse.Failure(concreteIndex, documentRequest.type(), documentRequest.id(), e);
+                    BulkItemResponse.Failure failure = new BulkItemResponse.Failure(concreteIndex.getName(), documentRequest.type(), documentRequest.id(), e);
                     BulkItemResponse bulkItemResponse = new BulkItemResponse(i, "delete", failure);
                     responses.set(i, bulkItemResponse);
                     // make sure the request gets never processed again
@@ -274,9 +276,9 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
 
             } else if (request instanceof UpdateRequest) {
                 try {
-                    TransportUpdateAction.resolveAndValidateRouting(metaData, concreteIndex, (UpdateRequest)request);
+                    TransportUpdateAction.resolveAndValidateRouting(metaData, concreteIndex.getName(), (UpdateRequest)request);
                 } catch(RoutingMissingException e) {
-                    BulkItemResponse.Failure failure = new BulkItemResponse.Failure(concreteIndex, documentRequest.type(), documentRequest.id(), e);
+                    BulkItemResponse.Failure failure = new BulkItemResponse.Failure(concreteIndex.getName(), documentRequest.type(), documentRequest.id(), e);
                     BulkItemResponse bulkItemResponse = new BulkItemResponse(i, "update", failure);
                     responses.set(i, bulkItemResponse);
                     // make sure the request gets never processed again
@@ -294,7 +296,7 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
             ActionRequest request = bulkRequest.requests.get(i);
             if (request instanceof IndexRequest) {
                 IndexRequest indexRequest = (IndexRequest) request;
-                String concreteIndex = concreteIndices.getConcreteIndex(indexRequest.index());
+                String concreteIndex = concreteIndices.getConcreteIndex(indexRequest.index()).getName();
                 ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex, indexRequest.type(), indexRequest.id(), indexRequest.routing()).shardId();
                 List<BulkItemRequest> list = requestsByShard.get(shardId);
                 if (list == null) {
@@ -304,7 +306,7 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
                 list.add(new BulkItemRequest(i, request));
             } else if (request instanceof DeleteRequest) {
                 DeleteRequest deleteRequest = (DeleteRequest) request;
-                String concreteIndex = concreteIndices.getConcreteIndex(deleteRequest.index());
+                String concreteIndex = concreteIndices.getConcreteIndex(deleteRequest.index()).getName();
                 ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex, deleteRequest.type(), deleteRequest.id(), deleteRequest.routing()).shardId();
                 List<BulkItemRequest> list = requestsByShard.get(shardId);
                 if (list == null) {
@@ -314,7 +316,7 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
                 list.add(new BulkItemRequest(i, request));
             } else if (request instanceof UpdateRequest) {
                 UpdateRequest updateRequest = (UpdateRequest) request;
-                String concreteIndex = concreteIndices.getConcreteIndex(updateRequest.index());
+                String concreteIndex = concreteIndices.getConcreteIndex(updateRequest.index()).getName();
                 ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex, updateRequest.type(), updateRequest.id(), updateRequest.routing()).shardId();
                 List<BulkItemRequest> list = requestsByShard.get(shardId);
                 if (list == null) {
@@ -356,18 +358,19 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
                 public void onFailure(Throwable e) {
                     // create failures for all relevant requests
                     for (BulkItemRequest request : requests) {
+                        final String indexName = concreteIndices.getConcreteIndex(request.index()).getName();
                         if (request.request() instanceof IndexRequest) {
                             IndexRequest indexRequest = (IndexRequest) request.request();
                             responses.set(request.id(), new BulkItemResponse(request.id(), indexRequest.opType().toString().toLowerCase(Locale.ENGLISH),
-                                    new BulkItemResponse.Failure(concreteIndices.getConcreteIndex(indexRequest.index()), indexRequest.type(), indexRequest.id(), e)));
+                                    new BulkItemResponse.Failure(indexName, indexRequest.type(), indexRequest.id(), e)));
                         } else if (request.request() instanceof DeleteRequest) {
                             DeleteRequest deleteRequest = (DeleteRequest) request.request();
                             responses.set(request.id(), new BulkItemResponse(request.id(), "delete",
-                                    new BulkItemResponse.Failure(concreteIndices.getConcreteIndex(deleteRequest.index()), deleteRequest.type(), deleteRequest.id(), e)));
+                                    new BulkItemResponse.Failure(indexName, deleteRequest.type(), deleteRequest.id(), e)));
                         } else if (request.request() instanceof UpdateRequest) {
                             UpdateRequest updateRequest = (UpdateRequest) request.request();
                             responses.set(request.id(), new BulkItemResponse(request.id(), "update",
-                                    new BulkItemResponse.Failure(concreteIndices.getConcreteIndex(updateRequest.index()), updateRequest.type(), updateRequest.id(), e)));
+                                    new BulkItemResponse.Failure(indexName, updateRequest.type(), updateRequest.id(), e)));
                         }
                     }
                     if (counter.decrementAndGet() == 0) {
@@ -385,7 +388,7 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
     private boolean addFailureIfIndexIsUnavailable(DocumentRequest request, BulkRequest bulkRequest, AtomicArray<BulkItemResponse> responses, int idx,
                                               final ConcreteIndices concreteIndices,
                                               final MetaData metaData) {
-        String concreteIndex = concreteIndices.getConcreteIndex(request.index());
+        Index concreteIndex = concreteIndices.getConcreteIndex(request.index());
         Exception unavailableException = null;
         if (concreteIndex == null) {
             try {
@@ -397,9 +400,9 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
             }
         }
         if (unavailableException == null) {
-            IndexMetaData indexMetaData = metaData.index(concreteIndex);
+            IndexMetaData indexMetaData = metaData.getIndexSafe(concreteIndex);
             if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
-                unavailableException = new IndexClosedException(metaData.index(request.index()).getIndex());
+                unavailableException = new IndexClosedException(concreteIndex);
             }
         }
         if (unavailableException != null) {
@@ -425,19 +428,19 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
     private static class ConcreteIndices  {
         private final ClusterState state;
         private final IndexNameExpressionResolver indexNameExpressionResolver;
-        private final Map<String, String> indices = new HashMap<>();
+        private final Map<String, Index> indices = new HashMap<>();
 
         ConcreteIndices(ClusterState state, IndexNameExpressionResolver indexNameExpressionResolver) {
             this.state = state;
             this.indexNameExpressionResolver = indexNameExpressionResolver;
         }
 
-        String getConcreteIndex(String indexOrAlias) {
+        Index getConcreteIndex(String indexOrAlias) {
             return indices.get(indexOrAlias);
         }
 
-        String resolveIfAbsent(DocumentRequest request) {
-            String concreteIndex = indices.get(request.index());
+        Index resolveIfAbsent(DocumentRequest request) {
+            Index concreteIndex = indices.get(request.index());
             if (concreteIndex == null) {
                 concreteIndex = indexNameExpressionResolver.concreteSingleIndex(state, request);
                 indices.put(request.index(), concreteIndex);
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
index 30f6b03a116..76402df8aa4 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
@@ -35,12 +35,12 @@ import org.elasticsearch.action.support.replication.TransportReplicationAction;
 import org.elasticsearch.action.update.UpdateHelper;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
@@ -74,17 +74,19 @@ public class TransportShardBulkAction extends TransportReplicationAction<BulkSha
 
     private final UpdateHelper updateHelper;
     private final boolean allowIdGeneration;
+    private final MappingUpdatedAction mappingUpdatedAction;
 
     @Inject
     public TransportShardBulkAction(Settings settings, TransportService transportService, ClusterService clusterService,
                                     IndicesService indicesService, ThreadPool threadPool, ShardStateAction shardStateAction,
                                     MappingUpdatedAction mappingUpdatedAction, UpdateHelper updateHelper, ActionFilters actionFilters,
                                     IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, ACTION_NAME, transportService, clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedAction,
-                actionFilters, indexNameExpressionResolver,
+        super(settings, ACTION_NAME, transportService, clusterService, indicesService, threadPool, shardStateAction,
+            actionFilters, indexNameExpressionResolver,
                 BulkShardRequest::new, BulkShardRequest::new, ThreadPool.Names.BULK);
         this.updateHelper = updateHelper;
         this.allowIdGeneration = settings.getAsBoolean("action.allow_id_generation", true);
+        this.mappingUpdatedAction = mappingUpdatedAction;
     }
 
     @Override
@@ -104,8 +106,9 @@ public class TransportShardBulkAction extends TransportReplicationAction<BulkSha
 
     @Override
     protected Tuple<BulkShardResponse, BulkShardRequest> shardOperationOnPrimary(MetaData metaData, BulkShardRequest request) {
-        final IndexService indexService = indicesService.indexServiceSafe(request.index());
-        final IndexShard indexShard = indexService.getShard(request.shardId().id());
+        ShardId shardId = request.shardId();
+        final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
+        final IndexShard indexShard = indexService.getShard(shardId.getId());
 
         long[] preVersions = new long[request.items().length];
         VersionType[] preVersionTypes = new VersionType[request.items().length];
diff --git a/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java b/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
index 3ded0ed8e83..783fab08bae 100644
--- a/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
+++ b/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
@@ -28,13 +28,12 @@ import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -60,10 +59,10 @@ public class TransportDeleteAction extends TransportReplicationAction<DeleteRequ
     public TransportDeleteAction(Settings settings, TransportService transportService, ClusterService clusterService,
                                  IndicesService indicesService, ThreadPool threadPool, ShardStateAction shardStateAction,
                                  TransportCreateIndexAction createIndexAction, ActionFilters actionFilters,
-                                 IndexNameExpressionResolver indexNameExpressionResolver, MappingUpdatedAction mappingUpdatedAction,
+                                 IndexNameExpressionResolver indexNameExpressionResolver,
                                  AutoCreateIndex autoCreateIndex) {
         super(settings, DeleteAction.NAME, transportService, clusterService, indicesService, threadPool, shardStateAction,
-                mappingUpdatedAction, actionFilters, indexNameExpressionResolver,
+            actionFilters, indexNameExpressionResolver,
                 DeleteRequest::new, DeleteRequest::new, ThreadPool.Names.INDEX);
         this.createIndexAction = createIndexAction;
         this.autoCreateIndex = autoCreateIndex;
diff --git a/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java b/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
index 5969e20065e..284e31406e2 100644
--- a/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
@@ -27,10 +27,10 @@ import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
 import org.elasticsearch.cache.recycler.PageCacheRecycler;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardIterator;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java
index 2893b16a251..f661c4a8b58 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java
@@ -28,6 +28,8 @@ import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentBuilderString;
+import org.elasticsearch.index.mapper.ip.IpFieldMapper;
+import org.joda.time.DateTime;
 
 import java.io.IOException;
 
@@ -146,6 +148,17 @@ public abstract class FieldStats<T extends Comparable<T>> implements Streamable,
      */
     protected abstract T valueOf(String value, String optionalFormat);
 
+    /**
+     * @param value
+     *            The value to be converted to a String
+     * @param optionalFormat
+     *            A string describing how to print the specified value. Whether
+     *            this parameter is supported depends on the implementation. If
+     *            optionalFormat is specified and the implementation doesn't
+     *            support it an {@link UnsupportedOperationException} is thrown
+     */
+    public abstract String stringValueOf(Object value, String optionalFormat);
+
     /**
      * Merges the provided stats into this stats instance.
      */
@@ -274,6 +287,18 @@ public abstract class FieldStats<T extends Comparable<T>> implements Streamable,
             return java.lang.Long.valueOf(value);
         }
 
+        @Override
+        public String stringValueOf(Object value, String optionalFormat) {
+            if (optionalFormat != null) {
+                throw new UnsupportedOperationException("custom format isn't supported");
+            }
+            if (value instanceof Number) {
+                return java.lang.Long.toString(((Number) value).longValue());
+            } else {
+                throw new IllegalArgumentException("value must be a Long: " + value);
+            }
+        }
+
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
@@ -327,6 +352,18 @@ public abstract class FieldStats<T extends Comparable<T>> implements Streamable,
             return java.lang.Float.valueOf(value);
         }
 
+        @Override
+        public String stringValueOf(Object value, String optionalFormat) {
+            if (optionalFormat != null) {
+                throw new UnsupportedOperationException("custom format isn't supported");
+            }
+            if (value instanceof Number) {
+                return java.lang.Float.toString(((Number) value).floatValue());
+            } else {
+                throw new IllegalArgumentException("value must be a Float: " + value);
+            }
+        }
+
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
@@ -380,6 +417,18 @@ public abstract class FieldStats<T extends Comparable<T>> implements Streamable,
             return java.lang.Double.valueOf(value);
         }
 
+        @Override
+        public String stringValueOf(Object value, String optionalFormat) {
+            if (optionalFormat != null) {
+                throw new UnsupportedOperationException("custom format isn't supported");
+            }
+            if (value instanceof Number) {
+                return java.lang.Double.toString(((Number) value).doubleValue());
+            } else {
+                throw new IllegalArgumentException("value must be a Double: " + value);
+            }
+        }
+
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
@@ -437,6 +486,18 @@ public abstract class FieldStats<T extends Comparable<T>> implements Streamable,
             return new BytesRef(value);
         }
 
+        @Override
+        public String stringValueOf(Object value, String optionalFormat) {
+            if (optionalFormat != null) {
+                throw new UnsupportedOperationException("custom format isn't supported");
+            }
+            if (value instanceof BytesRef) {
+                return ((BytesRef) value).utf8ToString();
+            } else {
+                throw new IllegalArgumentException("value must be a BytesRef: " + value);
+            }
+        }
+
         @Override
         protected void toInnerXContent(XContentBuilder builder) throws IOException {
             builder.field(Fields.MIN_VALUE, getMinValueAsString());
@@ -490,6 +551,25 @@ public abstract class FieldStats<T extends Comparable<T>> implements Streamable,
             return dateFormatter.parser().parseMillis(value);
         }
 
+        @Override
+        public String stringValueOf(Object value, String optionalFormat) {
+            FormatDateTimeFormatter dateFormatter = this.dateFormatter;
+            if (optionalFormat != null) {
+                dateFormatter = Joda.forPattern(optionalFormat);
+            }
+            long millis;
+            if (value instanceof java.lang.Long) {
+                millis = ((java.lang.Long) value).longValue();
+            } else if (value instanceof DateTime) {
+                millis = ((DateTime) value).getMillis();
+            } else if (value instanceof BytesRef) {
+                millis = dateFormatter.parser().parseMillis(((BytesRef) value).utf8ToString());
+            } else {
+                throw new IllegalArgumentException("value must be either a DateTime or a long: " + value);
+            }
+            return dateFormatter.printer().print(millis);
+        }
+
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
@@ -504,6 +584,28 @@ public abstract class FieldStats<T extends Comparable<T>> implements Streamable,
 
     }
 
+    public static class Ip extends Long {
+
+        public Ip(int maxDoc, int docCount, long sumDocFreq, long sumTotalTermFreq, long minValue, long maxValue) {
+            super(maxDoc, docCount, sumDocFreq, sumTotalTermFreq, minValue, maxValue);
+        }
+
+        protected Ip(int type, long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq, long minValue, long maxValue) {
+            super(type, maxDoc, docCount, sumDocFreq, sumTotalTermFreq, minValue, maxValue);
+        }
+
+        public Ip() {
+        }
+
+        @Override
+        public String stringValueOf(Object value, String optionalFormat) {
+            if (value instanceof BytesRef) {
+                return super.stringValueOf(IpFieldMapper.ipToLong(((BytesRef) value).utf8ToString()), optionalFormat);
+            }
+            return super.stringValueOf(value, optionalFormat);
+        }
+    }
+
     public static FieldStats read(StreamInput in) throws IOException {
         FieldStats stats;
         byte type = in.readByte();
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java
index 42360c5e0eb..de56a0f5c2e 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java
@@ -28,13 +28,13 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.TransportBroadcastAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexService;
diff --git a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
index 2d6bafc9623..b84493c4dca 100644
--- a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
+++ b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
@@ -22,12 +22,12 @@ package org.elasticsearch.action.get;
 import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.Preference;
 import org.elasticsearch.cluster.routing.ShardIterator;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexService;
diff --git a/core/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java b/core/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
index 7f5de65c614..1858ac8ba71 100644
--- a/core/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
+++ b/core/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
@@ -22,10 +22,10 @@ package org.elasticsearch.action.get;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
@@ -69,7 +69,7 @@ public class TransportMultiGetAction extends HandledTransportAction<MultiGetRequ
                 continue;
             }
             item.routing(clusterState.metaData().resolveIndexRouting(item.parent(), item.routing(), item.index()));
-            String concreteSingleIndex = indexNameExpressionResolver.concreteSingleIndex(clusterState, item);
+            String concreteSingleIndex = indexNameExpressionResolver.concreteSingleIndex(clusterState, item).getName();
             if (item.routing() == null && clusterState.getMetaData().routingRequired(concreteSingleIndex, item.type())) {
                 responses.set(i, new MultiGetItemResponse(null, new MultiGetResponse.Failure(concreteSingleIndex, item.type(), item.id(),
                         new IllegalArgumentException("routing is required for [" + concreteSingleIndex + "]/[" + item.type() + "]/[" + item.id() + "]"))));
diff --git a/core/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/core/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
index 1f07a5e79e6..05b672105f8 100644
--- a/core/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
+++ b/core/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
@@ -23,10 +23,10 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportActions;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardIterator;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexService;
diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
index 94614fb01d9..b792241b56c 100644
--- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
@@ -42,6 +42,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
@@ -584,14 +585,6 @@ public class IndexRequest extends ReplicationRequest<IndexRequest> implements Do
         return this.versionType;
     }
 
-    private Version getVersion(MetaData metaData, String concreteIndex) {
-        // this can go away in 3.0 but is here now for easy backporting - since in 2.x we need the version on the timestamp stuff
-        final IndexMetaData indexMetaData = metaData.getIndices().get(concreteIndex);
-        if (indexMetaData == null) {
-            throw new IndexNotFoundException(concreteIndex);
-        }
-        return Version.indexCreated(indexMetaData.getSettings());
-    }
 
     public void process(MetaData metaData, @Nullable MappingMetaData mappingMd, boolean allowIdGeneration, String concreteIndex) {
         // resolve the routing if needed
@@ -600,8 +593,7 @@ public class IndexRequest extends ReplicationRequest<IndexRequest> implements Do
         // resolve timestamp if provided externally
         if (timestamp != null) {
             timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp,
-                    mappingMd != null ? mappingMd.timestamp().dateTimeFormatter() : TimestampFieldMapper.Defaults.DATE_TIME_FORMATTER,
-                    getVersion(metaData, concreteIndex));
+                    mappingMd != null ? mappingMd.timestamp().dateTimeFormatter() : TimestampFieldMapper.Defaults.DATE_TIME_FORMATTER);
         }
         if (mappingMd != null) {
             // might as well check for routing here
@@ -645,7 +637,7 @@ public class IndexRequest extends ReplicationRequest<IndexRequest> implements Do
                 // assigned again because mappingMd and
                 // mappingMd#timestamp() are not null
                 assert mappingMd != null;
-                timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter(), getVersion(metaData, concreteIndex));
+                timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter());
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
index fdd018c51f2..9be8e4cef89 100644
--- a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
@@ -28,7 +28,6 @@ import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
@@ -36,6 +35,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -69,6 +69,7 @@ public class TransportIndexAction extends TransportReplicationAction<IndexReques
     private final TransportCreateIndexAction createIndexAction;
 
     private final ClusterService clusterService;
+    private final MappingUpdatedAction mappingUpdatedAction;
 
     @Inject
     public TransportIndexAction(Settings settings, TransportService transportService, ClusterService clusterService,
@@ -76,8 +77,9 @@ public class TransportIndexAction extends TransportReplicationAction<IndexReques
                                 TransportCreateIndexAction createIndexAction, MappingUpdatedAction mappingUpdatedAction,
                                 ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
                                 AutoCreateIndex autoCreateIndex) {
-        super(settings, IndexAction.NAME, transportService, clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedAction,
-                actionFilters, indexNameExpressionResolver, IndexRequest::new, IndexRequest::new, ThreadPool.Names.INDEX);
+        super(settings, IndexAction.NAME, transportService, clusterService, indicesService, threadPool, shardStateAction,
+            actionFilters, indexNameExpressionResolver, IndexRequest::new, IndexRequest::new, ThreadPool.Names.INDEX);
+        this.mappingUpdatedAction = mappingUpdatedAction;
         this.createIndexAction = createIndexAction;
         this.autoCreateIndex = autoCreateIndex;
         this.allowIdGeneration = settings.getAsBoolean("action.allow_id_generation", true);
@@ -143,7 +145,7 @@ public class TransportIndexAction extends TransportReplicationAction<IndexReques
     protected Tuple<IndexResponse, IndexRequest> shardOperationOnPrimary(MetaData metaData, IndexRequest request) throws Exception {
 
         // validate, if routing is required, that we got routing
-        IndexMetaData indexMetaData = metaData.index(request.shardId().getIndex());
+        IndexMetaData indexMetaData = metaData.getIndexSafe(request.shardId().getIndex());
         MappingMetaData mappingMd = indexMetaData.mappingOrDefault(request.type());
         if (mappingMd != null && mappingMd.routing().required()) {
             if (request.routing() == null) {
@@ -205,8 +207,7 @@ public class TransportIndexAction extends TransportReplicationAction<IndexReques
         Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
         final ShardId shardId = indexShard.shardId();
         if (update != null) {
-            final String indexName = shardId.getIndexName();
-            mappingUpdatedAction.updateMappingOnMasterSynchronously(indexName, request.type(), update);
+            mappingUpdatedAction.updateMappingOnMaster(shardId.getIndex(), request.type(), update);
             operation = prepareIndexOperationOnPrimary(request, indexShard);
             update = operation.parsedDoc().dynamicMappingsUpdate();
             if (update != null) {
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java b/core/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
index 6378eb5757b..74ce894b053 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
@@ -22,11 +22,11 @@ package org.elasticsearch.action.ingest;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.ingest.PipelineStore;
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java b/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java
index e762d0b8d33..8bac5c7b804 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java
@@ -22,11 +22,11 @@ package org.elasticsearch.action.ingest;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.ingest.PipelineStore;
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java b/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java
index 75555ccff7c..1eb9337c814 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java
@@ -112,7 +112,7 @@ public final class IngestActionFilter extends AbstractComponent implements Actio
                 logger.error("failed to execute pipeline for a bulk request", throwable);
                 listener.onFailure(throwable);
             } else {
-                long ingestTookInMillis = TimeUnit.MILLISECONDS.convert(System.nanoTime() - ingestStartTimeInNanos, TimeUnit.NANOSECONDS);
+                long ingestTookInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ingestStartTimeInNanos);
                 BulkRequest bulkRequest = bulkRequestModifier.getBulkRequest();
                 ActionListener<BulkResponse> actionListener = bulkRequestModifier.wrapActionListenerIfNeeded(ingestTookInMillis, listener);
                 if (bulkRequest.requests().isEmpty()) {
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/IngestProxyActionFilter.java b/core/src/main/java/org/elasticsearch/action/ingest/IngestProxyActionFilter.java
index 39a4b1fa4e8..62716c6dc0d 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/IngestProxyActionFilter.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/IngestProxyActionFilter.java
@@ -29,9 +29,9 @@ import org.elasticsearch.action.index.IndexAction;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.ActionFilter;
 import org.elasticsearch.action.support.ActionFilterChain;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
index aafd9ee75a4..e1a34413e2c 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
@@ -26,12 +26,12 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.ingest.PipelineStore;
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java
index 30efbe1b0fa..bc40a8368f0 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java
@@ -23,13 +23,14 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.ingest.core.Pipeline;
-import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.ingest.core.CompoundProcessor;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.elasticsearch.ingest.processor.TrackingResultProcessor.decorate;
+
 class SimulateExecutionService {
 
     private static final String THREAD_POOL_NAME = ThreadPool.Names.MANAGEMENT;
@@ -40,40 +41,16 @@ class SimulateExecutionService {
         this.threadPool = threadPool;
     }
 
-    void executeVerboseDocument(Processor processor, IngestDocument ingestDocument, List<SimulateProcessorResult> processorResultList) throws Exception {
-        if (processor instanceof CompoundProcessor) {
-            CompoundProcessor cp = (CompoundProcessor) processor;
-            try {
-                for (Processor p : cp.getProcessors()) {
-                    executeVerboseDocument(p, ingestDocument, processorResultList);
-                }
-            } catch (Exception e) {
-                for (Processor p : cp.getOnFailureProcessors()) {
-                    executeVerboseDocument(p, ingestDocument, processorResultList);
-                }
-            }
-        } else {
-            try {
-                processor.execute(ingestDocument);
-                processorResultList.add(new SimulateProcessorResult(processor.getTag(), new IngestDocument(ingestDocument)));
-            } catch (Exception e) {
-                processorResultList.add(new SimulateProcessorResult(processor.getTag(), e));
-                throw e;
-            }
-        }
-    }
-
     SimulateDocumentResult executeDocument(Pipeline pipeline, IngestDocument ingestDocument, boolean verbose) {
         if (verbose) {
             List<SimulateProcessorResult> processorResultList = new ArrayList<>();
-            IngestDocument currentIngestDocument = new IngestDocument(ingestDocument);
-            CompoundProcessor pipelineProcessor = new CompoundProcessor(pipeline.getProcessors(), pipeline.getOnFailureProcessors());
+            CompoundProcessor verbosePipelineProcessor = decorate(pipeline.getCompoundProcessor(), processorResultList);
             try {
-                executeVerboseDocument(pipelineProcessor, currentIngestDocument, processorResultList);
+                verbosePipelineProcessor.execute(ingestDocument);
+                return new SimulateDocumentVerboseResult(processorResultList);
             } catch (Exception e) {
-                return new SimulateDocumentBaseResult(e);
+                return new SimulateDocumentVerboseResult(processorResultList);
             }
-            return new SimulateDocumentVerboseResult(processorResultList);
         } else {
             try {
                 pipeline.execute(ingestDocument);
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/MultiPercolateRequest.java b/core/src/main/java/org/elasticsearch/action/percolate/MultiPercolateRequest.java
index 74537379d1d..ac49fed763a 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/MultiPercolateRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/percolate/MultiPercolateRequest.java
@@ -163,11 +163,7 @@ public class MultiPercolateRequest extends ActionRequest<MultiPercolateRequest>
 
     @Override
     public List<? extends IndicesRequest> subRequests() {
-        List<IndicesRequest> indicesRequests = new ArrayList<>();
-        for (PercolateRequest percolateRequest : this.requests) {
-            indicesRequests.addAll(percolateRequest.subRequests());
-        }
-        return indicesRequests;
+        return requests;
     }
 
     private void parsePercolateAction(XContentParser parser, PercolateRequest percolateRequest, boolean allowExplicitIndex) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequest.java b/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequest.java
index e69da6bf519..c9887cba03f 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequest.java
@@ -19,10 +19,12 @@
 package org.elasticsearch.action.percolate;
 
 import org.elasticsearch.ElasticsearchGenerationException;
+import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.CompositeIndicesRequest;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.broadcast.BroadcastRequest;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -43,49 +45,37 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
 /**
  * A request to execute a percolate operation.
  */
-public class PercolateRequest extends BroadcastRequest<PercolateRequest> implements CompositeIndicesRequest {
+public class PercolateRequest extends ActionRequest<PercolateRequest> implements IndicesRequest.Replaceable {
 
+    protected String[] indices;
+    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
     private String documentType;
     private String routing;
     private String preference;
-    private GetRequest getRequest;
     private boolean onlyCount;
 
+    private GetRequest getRequest;
     private BytesReference source;
 
-    private BytesReference docSource;
-
-    // Used internally in order to compute tookInMillis, TransportBroadcastAction itself doesn't allow
-    // to hold it temporarily in an easy way
-    long startTime;
-
-    /**
-     * Constructor only for internal usage.
-     */
-    public PercolateRequest() {
+    public String[] indices() {
+        return indices;
     }
 
-    PercolateRequest(PercolateRequest request, BytesReference docSource) {
-        this.indices = request.indices();
-        this.documentType = request.documentType();
-        this.routing = request.routing();
-        this.preference = request.preference();
-        this.source = request.source;
-        this.docSource = docSource;
-        this.onlyCount = request.onlyCount;
-        this.startTime = request.startTime;
+    public final PercolateRequest indices(String... indices) {
+        this.indices = indices;
+        return this;
     }
 
-    @Override
-    public List<? extends IndicesRequest> subRequests() {
-        List<IndicesRequest> requests = new ArrayList<>();
-        requests.add(this);
-        if (getRequest != null) {
-            requests.add(getRequest);
-        }
-        return requests;
+    public IndicesOptions indicesOptions() {
+        return indicesOptions;
     }
 
+    public PercolateRequest indicesOptions(IndicesOptions indicesOptions) {
+        this.indicesOptions = indicesOptions;
+        return this;
+    }
+
+
     /**
      * Getter for {@link #documentType(String)}
      */
@@ -244,13 +234,9 @@ public class PercolateRequest extends BroadcastRequest<PercolateRequest> impleme
         return this;
     }
 
-    BytesReference docSource() {
-        return docSource;
-    }
-
     @Override
     public ActionRequestValidationException validate() {
-        ActionRequestValidationException validationException = super.validate();
+        ActionRequestValidationException validationException = null;
         if (documentType == null) {
             validationException = addValidationError("type is missing", validationException);
         }
@@ -266,12 +252,12 @@ public class PercolateRequest extends BroadcastRequest<PercolateRequest> impleme
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        startTime = in.readVLong();
+        indices = in.readStringArray();
+        indicesOptions = IndicesOptions.readIndicesOptions(in);
         documentType = in.readString();
         routing = in.readOptionalString();
         preference = in.readOptionalString();
         source = in.readBytesReference();
-        docSource = in.readBytesReference();
         if (in.readBoolean()) {
             getRequest = new GetRequest();
             getRequest.readFrom(in);
@@ -282,12 +268,12 @@ public class PercolateRequest extends BroadcastRequest<PercolateRequest> impleme
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeVLong(startTime);
+        out.writeStringArrayNullable(indices);
+        indicesOptions.writeIndicesOptions(out);
         out.writeString(documentType);
         out.writeOptionalString(routing);
         out.writeOptionalString(preference);
         out.writeBytesReference(source);
-        out.writeBytesReference(docSource);
         if (getRequest != null) {
             out.writeBoolean(true);
             getRequest.writeTo(out);
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java
index 472938cfbf1..9286601da69 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java
@@ -18,8 +18,9 @@
  */
 package org.elasticsearch.action.percolate;
 
+import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.action.support.broadcast.BroadcastOperationRequestBuilder;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -36,7 +37,7 @@ import java.util.Map;
 /**
  * A builder the easy to use of defining a percolate request.
  */
-public class PercolateRequestBuilder extends BroadcastOperationRequestBuilder<PercolateRequest, PercolateResponse, PercolateRequestBuilder> {
+public class PercolateRequestBuilder extends ActionRequestBuilder<PercolateRequest, PercolateResponse, PercolateRequestBuilder> {
 
     private PercolateSourceBuilder sourceBuilder;
 
@@ -44,6 +45,16 @@ public class PercolateRequestBuilder extends BroadcastOperationRequestBuilder<Pe
         super(client, action, new PercolateRequest());
     }
 
+    public PercolateRequestBuilder setIndices(String... indices) {
+        request.indices(indices);
+        return this;
+    }
+
+    public PercolateRequestBuilder setIndicesOptions(IndicesOptions indicesOptions) {
+        request.indicesOptions(indicesOptions);
+        return this;
+    }
+
     /**
      * Sets the type of the document to percolate. This is important as it selects the mapping to be used to parse
      * the document.
@@ -115,7 +126,7 @@ public class PercolateRequestBuilder extends BroadcastOperationRequestBuilder<Pe
     /**
      * Delegates to {@link PercolateSourceBuilder#addSort(SortBuilder)}
      */
-    public PercolateRequestBuilder addSort(SortBuilder sort) {
+    public PercolateRequestBuilder addSort(SortBuilder<?> sort) {
         sourceBuilder().addSort(sort);
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateResponse.java b/core/src/main/java/org/elasticsearch/action/percolate/PercolateResponse.java
index 13c2526a7a3..9400cece8b6 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/percolate/PercolateResponse.java
@@ -29,7 +29,6 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentBuilderString;
-import org.elasticsearch.percolator.PercolatorService;
 import org.elasticsearch.rest.action.support.RestActions;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.highlight.HighlightField;
@@ -47,6 +46,8 @@ import java.util.Map;
 public class PercolateResponse extends BroadcastResponse implements Iterable<PercolateResponse.Match>, ToXContent {
 
     public static final Match[] EMPTY = new Match[0];
+    // PercolatorQuery emits this score if no 'query' is defined in the percolate request
+    public final static float NO_SCORE = 0.0f;
 
     private long tookInMillis;
     private Match[] matches;
@@ -65,15 +66,6 @@ public class PercolateResponse extends BroadcastResponse implements Iterable<Per
         this.aggregations = aggregations;
     }
 
-    PercolateResponse(int totalShards, int successfulShards, int failedShards, List<ShardOperationFailedException> shardFailures, long tookInMillis, Match[] matches) {
-        super(totalShards, successfulShards, failedShards, shardFailures);
-        if (tookInMillis < 0) {
-            throw new IllegalArgumentException("tookInMillis must be positive but was: " + tookInMillis);
-        }
-        this.tookInMillis = tookInMillis;
-        this.matches = matches;
-    }
-
     PercolateResponse() {
     }
 
@@ -136,10 +128,10 @@ public class PercolateResponse extends BroadcastResponse implements Iterable<Per
                     builder.field(Fields._INDEX, match.getIndex());
                     builder.field(Fields._ID, match.getId());
                     float score = match.getScore();
-                    if (score != PercolatorService.NO_SCORE) {
+                    if (score != NO_SCORE) {
                         builder.field(Fields._SCORE, match.getScore());
                     }
-                    if (match.getHighlightFields() != null) {
+                    if (match.getHighlightFields().isEmpty() == false) {
                         builder.startObject(Fields.HIGHLIGHT);
                         for (HighlightField field : match.getHighlightFields().values()) {
                             builder.field(field.name());
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateShardRequest.java b/core/src/main/java/org/elasticsearch/action/percolate/PercolateShardRequest.java
deleted file mode 100644
index 74d531df0c4..00000000000
--- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateShardRequest.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.percolate;
-
-import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.action.support.broadcast.BroadcastShardRequest;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.index.shard.ShardId;
-
-import java.io.IOException;
-
-/**
- */
-public class PercolateShardRequest extends BroadcastShardRequest {
-
-    private String documentType;
-    private BytesReference source;
-    private BytesReference docSource;
-    private boolean onlyCount;
-    private int numberOfShards;
-    private long startTime;
-
-    public PercolateShardRequest() {
-    }
-
-    PercolateShardRequest(ShardId shardId, int numberOfShards, PercolateRequest request) {
-        super(shardId, request);
-        this.documentType = request.documentType();
-        this.source = request.source();
-        this.docSource = request.docSource();
-        this.onlyCount = request.onlyCount();
-        this.numberOfShards = numberOfShards;
-        this.startTime = request.startTime;
-    }
-
-    PercolateShardRequest(ShardId shardId, PercolateRequest request) {
-        super(shardId, request);
-        this.documentType = request.documentType();
-        this.source = request.source();
-        this.docSource = request.docSource();
-        this.onlyCount = request.onlyCount();
-        this.startTime = request.startTime;
-    }
-
-    public String documentType() {
-        return documentType;
-    }
-
-    public BytesReference source() {
-        return source;
-    }
-
-    public BytesReference docSource() {
-        return docSource;
-    }
-
-    public boolean onlyCount() {
-        return onlyCount;
-    }
-
-    public void documentType(String documentType) {
-        this.documentType = documentType;
-    }
-
-    public void source(BytesReference source) {
-        this.source = source;
-    }
-
-    public void docSource(BytesReference docSource) {
-        this.docSource = docSource;
-    }
-
-    void onlyCount(boolean onlyCount) {
-        this.onlyCount = onlyCount;
-    }
-
-    public int getNumberOfShards() {
-        return numberOfShards;
-    }
-
-    public long getStartTime() {
-        return startTime;
-    }
-
-    OriginalIndices originalIndices() {
-        return originalIndices;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        documentType = in.readString();
-        source = in.readBytesReference();
-        docSource = in.readBytesReference();
-        onlyCount = in.readBoolean();
-        numberOfShards = in.readVInt();
-        startTime = in.readLong(); // no vlong, this can be negative!
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeString(documentType);
-        out.writeBytesReference(source);
-        out.writeBytesReference(docSource);
-        out.writeBoolean(onlyCount);
-        out.writeVInt(numberOfShards);
-        out.writeLong(startTime);
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateShardResponse.java b/core/src/main/java/org/elasticsearch/action/percolate/PercolateShardResponse.java
deleted file mode 100644
index e3fc737c2cc..00000000000
--- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateShardResponse.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.action.percolate;
-
-import org.apache.lucene.search.TopDocs;
-import org.elasticsearch.action.support.broadcast.BroadcastShardResponse;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.percolator.PercolateContext;
-import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorStreams;
-import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator;
-import org.elasticsearch.search.highlight.HighlightField;
-import org.elasticsearch.search.query.QuerySearchResult;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- */
-public class PercolateShardResponse extends BroadcastShardResponse {
-
-    private TopDocs topDocs;
-    private Map<Integer, String> ids;
-    private Map<Integer, Map<String, HighlightField>> hls;
-    private boolean onlyCount;
-    private int requestedSize;
-
-    private InternalAggregations aggregations;
-    private List<SiblingPipelineAggregator> pipelineAggregators;
-
-    PercolateShardResponse() {
-    }
-
-    public PercolateShardResponse(TopDocs topDocs, Map<Integer, String> ids, Map<Integer, Map<String, HighlightField>> hls, PercolateContext context) {
-        super(context.indexShard().shardId());
-        this.topDocs = topDocs;
-        this.ids = ids;
-        this.hls = hls;
-        this.onlyCount = context.isOnlyCount();
-        this.requestedSize = context.size();
-        QuerySearchResult result = context.queryResult();
-        if (result != null) {
-            if (result.aggregations() != null) {
-                this.aggregations = (InternalAggregations) result.aggregations();
-            }
-            this.pipelineAggregators = result.pipelineAggregators();
-        }
-    }
-
-    public TopDocs topDocs() {
-        return topDocs;
-    }
-
-    /**
-     * Returns per match the percolator query id. The key is the Lucene docId of the matching percolator query.
-     */
-    public Map<Integer, String> ids() {
-        return ids;
-    }
-
-    public int requestedSize() {
-        return requestedSize;
-    }
-
-    /**
-     * Returns per match the highlight snippets. The key is the Lucene docId of the matching percolator query.
-     */
-    public Map<Integer, Map<String, HighlightField>> hls() {
-        return hls;
-    }
-
-    public InternalAggregations aggregations() {
-        return aggregations;
-    }
-
-    public List<SiblingPipelineAggregator> pipelineAggregators() {
-        return pipelineAggregators;
-    }
-
-    public boolean onlyCount() {
-        return onlyCount;
-    }
-
-    public boolean isEmpty() {
-        return topDocs.totalHits == 0;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        onlyCount = in.readBoolean();
-        requestedSize = in.readVInt();
-        topDocs = Lucene.readTopDocs(in);
-        int size = in.readVInt();
-        ids = new HashMap<>(size);
-        for (int i = 0; i < size; i++) {
-            ids.put(in.readVInt(), in.readString());
-        }
-        size = in.readVInt();
-        hls = new HashMap<>(size);
-        for (int i = 0; i < size; i++) {
-            int docId = in.readVInt();
-            int mSize = in.readVInt();
-            Map<String, HighlightField> fields = new HashMap<>();
-            for (int j = 0; j < mSize; j++) {
-                fields.put(in.readString(), HighlightField.readHighlightField(in));
-            }
-            hls.put(docId, fields);
-        }
-        aggregations = InternalAggregations.readOptionalAggregations(in);
-        if (in.readBoolean()) {
-            int pipelineAggregatorsSize = in.readVInt();
-            List<SiblingPipelineAggregator> pipelineAggregators = new ArrayList<>(pipelineAggregatorsSize);
-            for (int i = 0; i < pipelineAggregatorsSize; i++) {
-                BytesReference type = in.readBytesReference();
-                PipelineAggregator pipelineAggregator = PipelineAggregatorStreams.stream(type).readResult(in);
-                pipelineAggregators.add((SiblingPipelineAggregator) pipelineAggregator);
-            }
-            this.pipelineAggregators = pipelineAggregators;
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeBoolean(onlyCount);
-        out.writeVLong(requestedSize);
-        Lucene.writeTopDocs(out, topDocs);
-        out.writeVInt(ids.size());
-        for (Map.Entry<Integer, String> entry : ids.entrySet()) {
-            out.writeVInt(entry.getKey());
-            out.writeString(entry.getValue());
-        }
-        out.writeVInt(hls.size());
-        for (Map.Entry<Integer, Map<String, HighlightField>> entry1 : hls.entrySet()) {
-            out.writeVInt(entry1.getKey());
-            out.writeVInt(entry1.getValue().size());
-            for (Map.Entry<String, HighlightField> entry2 : entry1.getValue().entrySet()) {
-                out.writeString(entry2.getKey());
-                entry2.getValue().writeTo(out);
-            }
-        }
-        out.writeOptionalStreamable(aggregations);
-        if (pipelineAggregators == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            out.writeVInt(pipelineAggregators.size());
-            for (PipelineAggregator pipelineAggregator : pipelineAggregators) {
-                out.writeBytesReference(pipelineAggregator.type().stream());
-                pipelineAggregator.writeTo(out);
-            }
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java b/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java
index d687d48fb0c..5a5924f7883 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java
@@ -48,13 +48,13 @@ import java.util.Map;
 public class PercolateSourceBuilder extends ToXContentToBytes {
 
     private DocBuilder docBuilder;
-    private QueryBuilder queryBuilder;
+    private QueryBuilder<?> queryBuilder;
     private Integer size;
-    private List<SortBuilder> sorts;
+    private List<SortBuilder<?>> sorts;
     private Boolean trackScores;
     private HighlightBuilder highlightBuilder;
     private List<AggregatorBuilder<?>> aggregationBuilders;
-    private List<PipelineAggregatorBuilder> pipelineAggregationBuilders;
+    private List<PipelineAggregatorBuilder<?>> pipelineAggregationBuilders;
 
     /**
      * Sets the document to run the percolate queries against.
@@ -68,7 +68,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes {
      * Sets a query to reduce the number of percolate queries to be evaluated and score the queries that match based
      * on this query.
      */
-    public PercolateSourceBuilder setQueryBuilder(QueryBuilder queryBuilder) {
+    public PercolateSourceBuilder setQueryBuilder(QueryBuilder<?> queryBuilder) {
         this.queryBuilder = queryBuilder;
         return this;
     }
@@ -98,7 +98,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes {
      *
      * By default the matching percolator queries are returned in an undefined order.
      */
-    public PercolateSourceBuilder addSort(SortBuilder sort) {
+    public PercolateSourceBuilder addSort(SortBuilder<?> sort) {
         if (sorts == null) {
             sorts = new ArrayList<>();
         }
@@ -137,7 +137,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes {
     /**
      * Add an aggregation definition.
      */
-    public PercolateSourceBuilder addAggregation(PipelineAggregatorBuilder aggregationBuilder) {
+    public PercolateSourceBuilder addAggregation(PipelineAggregatorBuilder<?> aggregationBuilder) {
         if (pipelineAggregationBuilders == null) {
             pipelineAggregationBuilders = new ArrayList<>();
         }
@@ -160,10 +160,8 @@ public class PercolateSourceBuilder extends ToXContentToBytes {
         }
         if (sorts != null) {
             builder.startArray("sort");
-            for (SortBuilder sort : sorts) {
-                builder.startObject();
+            for (SortBuilder<?> sort : sorts) {
                 sort.toXContent(builder, params);
-                builder.endObject();
             }
             builder.endArray();
         }
@@ -182,7 +180,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes {
                 }
             }
             if (pipelineAggregationBuilders != null) {
-                for (PipelineAggregatorBuilder aggregation : pipelineAggregationBuilders) {
+                for (PipelineAggregatorBuilder<?> aggregation : pipelineAggregationBuilders) {
                     aggregation.toXContent(builder, params);
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/TransportMultiPercolateAction.java b/core/src/main/java/org/elasticsearch/action/percolate/TransportMultiPercolateAction.java
index d86d91c654e..bf0d79d884e 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/TransportMultiPercolateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/percolate/TransportMultiPercolateAction.java
@@ -19,114 +19,91 @@
 
 package org.elasticsearch.action.percolate;
 
-import com.carrotsearch.hppc.IntArrayList;
+import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.UnavailableShardsException;
 import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.get.MultiGetItemResponse;
 import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.get.MultiGetResponse;
-import org.elasticsearch.action.get.TransportMultiGetAction;
+import org.elasticsearch.action.search.MultiSearchRequest;
+import org.elasticsearch.action.search.MultiSearchResponse;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
-import org.elasticsearch.cluster.ClusterService;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
-import org.elasticsearch.cluster.routing.ShardIterator;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.AtomicArray;
-import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.index.engine.DocumentMissingException;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.percolator.PercolatorService;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReferenceArray;
 
-/**
- */
 public class TransportMultiPercolateAction extends HandledTransportAction<MultiPercolateRequest, MultiPercolateResponse> {
 
-    private final ClusterService clusterService;
-    private final PercolatorService percolatorService;
-
-    private final TransportMultiGetAction multiGetAction;
-    private final TransportShardMultiPercolateAction shardMultiPercolateAction;
+    private final Client client;
+    private final ParseFieldMatcher parseFieldMatcher;
+    private final IndicesQueriesRegistry queryRegistry;
+    private final AggregatorParsers aggParsers;
 
     @Inject
-    public TransportMultiPercolateAction(Settings settings, ThreadPool threadPool, TransportShardMultiPercolateAction shardMultiPercolateAction,
-                                         ClusterService clusterService, TransportService transportService, PercolatorService percolatorService,
-                                         TransportMultiGetAction multiGetAction, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+    public TransportMultiPercolateAction(Settings settings, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters,
+                                         IndexNameExpressionResolver indexNameExpressionResolver, Client client, IndicesQueriesRegistry queryRegistry,
+                                         AggregatorParsers aggParsers) {
         super(settings, MultiPercolateAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, MultiPercolateRequest::new);
-        this.shardMultiPercolateAction = shardMultiPercolateAction;
-        this.clusterService = clusterService;
-        this.percolatorService = percolatorService;
-        this.multiGetAction = multiGetAction;
+        this.client = client;
+        this.aggParsers = aggParsers;
+        this.parseFieldMatcher = new ParseFieldMatcher(settings);
+        this.queryRegistry = queryRegistry;
     }
 
     @Override
-    protected void doExecute(final MultiPercolateRequest request, final ActionListener<MultiPercolateResponse> listener) {
-        final ClusterState clusterState = clusterService.state();
-        clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
-
-        final List<Object> percolateRequests = new ArrayList<>(request.requests().size());
-        // Can have a mixture of percolate requests. (normal percolate requests & percolate existing doc),
-        // so we need to keep track for what percolate request we had a get request
-        final IntArrayList getRequestSlots = new IntArrayList();
-        List<GetRequest> existingDocsRequests = new ArrayList<>();
-        for (int slot = 0;  slot < request.requests().size(); slot++) {
-            PercolateRequest percolateRequest = request.requests().get(slot);
-            percolateRequest.startTime = System.currentTimeMillis();
-            percolateRequests.add(percolateRequest);
-            if (percolateRequest.getRequest() != null) {
-                existingDocsRequests.add(percolateRequest.getRequest());
-                getRequestSlots.add(slot);
+    protected void doExecute(MultiPercolateRequest request, ActionListener<MultiPercolateResponse> listener) {
+        List<Tuple<Integer, GetRequest>> getRequests = new ArrayList<>();
+        for (int i = 0; i < request.requests().size(); i++) {
+            GetRequest getRequest = request.requests().get(i).getRequest();
+            if (getRequest != null) {
+                getRequests.add(new Tuple<>(i, getRequest));
             }
         }
-
-        if (!existingDocsRequests.isEmpty()) {
-            final MultiGetRequest multiGetRequest = new MultiGetRequest();
-            for (GetRequest getRequest : existingDocsRequests) {
-                multiGetRequest.add(
-                        new MultiGetRequest.Item(getRequest.index(), getRequest.type(), getRequest.id())
-                        .routing(getRequest.routing())
-                );
+        if (getRequests.isEmpty()) {
+            innerDoExecute(request, listener, Collections.emptyMap(), new HashMap<>());
+        } else {
+            MultiGetRequest multiGetRequest = new MultiGetRequest();
+            for (Tuple<Integer, GetRequest> tuple : getRequests) {
+                GetRequest getRequest = tuple.v2();
+                multiGetRequest.add(new MultiGetRequest.Item(getRequest.index(), getRequest.type(), getRequest.id()));
             }
-
-            multiGetAction.execute(multiGetRequest, new ActionListener<MultiGetResponse>() {
-
+            client.multiGet(multiGetRequest, new ActionListener<MultiGetResponse>() {
                 @Override
-                public void onResponse(MultiGetResponse multiGetItemResponses) {
-                    for (int i = 0; i < multiGetItemResponses.getResponses().length; i++) {
-                        MultiGetItemResponse itemResponse = multiGetItemResponses.getResponses()[i];
-                        int slot = getRequestSlots.get(i);
-                        if (!itemResponse.isFailed()) {
-                            GetResponse getResponse = itemResponse.getResponse();
-                            if (getResponse.isExists()) {
-                                PercolateRequest originalRequest = (PercolateRequest) percolateRequests.get(slot);
-                                percolateRequests.set(slot, new PercolateRequest(originalRequest, getResponse.getSourceAsBytesRef()));
-                            } else {
-                                logger.trace("mpercolate existing doc, item[{}] doesn't exist", slot);
-                                percolateRequests.set(slot, new DocumentMissingException(null, getResponse.getType(), getResponse.getId()));
-                            }
+                public void onResponse(MultiGetResponse response) {
+                    Map<Integer, BytesReference> getResponseSources = new HashMap<>(response.getResponses().length);
+                    Map<Integer, MultiPercolateResponse.Item> preFailures = new HashMap<>();
+                    for (int i = 0; i < response.getResponses().length; i++) {
+                        MultiGetItemResponse itemResponse = response.getResponses()[i];
+                        int originalSlot = getRequests.get(i).v1();
+                        if (itemResponse.isFailed()) {
+                            preFailures.put(originalSlot, new MultiPercolateResponse.Item(itemResponse.getFailure().getFailure()));
                         } else {
-                            logger.trace("mpercolate existing doc, item[{}] failure {}", slot, itemResponse.getFailure());
-                            percolateRequests.set(slot, itemResponse.getFailure());
+                            if (itemResponse.getResponse().isExists()) {
+                                getResponseSources.put(originalSlot, itemResponse.getResponse().getSourceAsBytesRef());
+                            } else {
+                                GetRequest getRequest = getRequests.get(i).v2();
+                                preFailures.put(originalSlot, new MultiPercolateResponse.Item(new ResourceNotFoundException("percolate document [{}/{}/{}] doesn't exist", getRequest.index(), getRequest.type(), getRequest.id())));
+                            }
                         }
                     }
-                    new ASyncAction(request, percolateRequests, listener, clusterState).run();
+                    innerDoExecute(request, listener, getResponseSources, preFailures);
                 }
 
                 @Override
@@ -134,200 +111,81 @@ public class TransportMultiPercolateAction extends HandledTransportAction<MultiP
                     listener.onFailure(e);
                 }
             });
-        } else {
-            new ASyncAction(request, percolateRequests, listener, clusterState).run();
         }
-
     }
 
-    private final class ASyncAction {
-
-        final ActionListener<MultiPercolateResponse> finalListener;
-        final Map<ShardId, TransportShardMultiPercolateAction.Request> requestsByShard;
-        final MultiPercolateRequest multiPercolateRequest;
-        final List<Object> percolateRequests;
-
-        final Map<ShardId, IntArrayList> shardToSlots;
-        final AtomicInteger expectedOperations;
-        final AtomicArray<Object> reducedResponses;
-        final AtomicReferenceArray<AtomicInteger> expectedOperationsPerItem;
-        final AtomicReferenceArray<AtomicReferenceArray> responsesByItemAndShard;
-
-        ASyncAction(MultiPercolateRequest multiPercolateRequest, List<Object> percolateRequests, ActionListener<MultiPercolateResponse> finalListener, ClusterState clusterState) {
-            this.finalListener = finalListener;
-            this.multiPercolateRequest = multiPercolateRequest;
-            this.percolateRequests = percolateRequests;
-            responsesByItemAndShard = new AtomicReferenceArray<>(percolateRequests.size());
-            expectedOperationsPerItem = new AtomicReferenceArray<>(percolateRequests.size());
-            reducedResponses = new AtomicArray<>(percolateRequests.size());
-
-            // Resolving concrete indices and routing and grouping the requests by shard
-            requestsByShard = new HashMap<>();
-            // Keep track what slots belong to what shard, in case a request to a shard fails on all copies
-            shardToSlots = new HashMap<>();
-            int expectedResults = 0;
-            for (int slot = 0;  slot < percolateRequests.size(); slot++) {
-                Object element = percolateRequests.get(slot);
-                assert element != null;
-                if (element instanceof PercolateRequest) {
-                    PercolateRequest percolateRequest = (PercolateRequest) element;
-                    String[] concreteIndices;
-                    try {
-                         concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, percolateRequest);
-                    } catch (IndexNotFoundException e) {
-                        reducedResponses.set(slot, e);
-                        responsesByItemAndShard.set(slot, new AtomicReferenceArray(0));
-                        expectedOperationsPerItem.set(slot, new AtomicInteger(0));
-                        continue;
-                    }
-                    Map<String, Set<String>> routing = indexNameExpressionResolver.resolveSearchRouting(clusterState, percolateRequest.routing(), percolateRequest.indices());
-                    // TODO: I only need shardIds, ShardIterator(ShardRouting) is only needed in TransportShardMultiPercolateAction
-                    GroupShardsIterator shards = clusterService.operationRouting().searchShards(
-                            clusterState, concreteIndices, routing, percolateRequest.preference()
-                    );
-                    if (shards.size() == 0) {
-                        reducedResponses.set(slot, new UnavailableShardsException(null, "No shards available"));
-                        responsesByItemAndShard.set(slot, new AtomicReferenceArray(0));
-                        expectedOperationsPerItem.set(slot, new AtomicInteger(0));
-                        continue;
-                    }
-
-                    // The shard id is used as index in the atomic ref array, so we need to find out how many shards there are regardless of routing:
-                    int numShards = clusterService.operationRouting().searchShardsCount(clusterState, concreteIndices, null);
-                    responsesByItemAndShard.set(slot, new AtomicReferenceArray(numShards));
-                    expectedOperationsPerItem.set(slot, new AtomicInteger(shards.size()));
-                    for (ShardIterator shard : shards) {
-                        ShardId shardId = shard.shardId();
-                        TransportShardMultiPercolateAction.Request requests = requestsByShard.get(shardId);
-                        if (requests == null) {
-                            requestsByShard.put(shardId, requests = new TransportShardMultiPercolateAction.Request(shardId.getIndexName(), shardId.getId(), percolateRequest.preference()));
-                        }
-                        logger.trace("Adding shard[{}] percolate request for item[{}]", shardId, slot);
-                        requests.add(new TransportShardMultiPercolateAction.Request.Item(slot, new PercolateShardRequest(shardId, percolateRequest)));
-
-                        IntArrayList items = shardToSlots.get(shardId);
-                        if (items == null) {
-                            shardToSlots.put(shardId, items = new IntArrayList());
-                        }
-                        items.add(slot);
-                    }
-                    expectedResults++;
-                } else if (element instanceof Throwable || element instanceof MultiGetResponse.Failure) {
-                    logger.trace("item[{}] won't be executed, reason: {}", slot, element);
-                    reducedResponses.set(slot, element);
-                    responsesByItemAndShard.set(slot, new AtomicReferenceArray(0));
-                    expectedOperationsPerItem.set(slot, new AtomicInteger(0));
-                }
-            }
-            expectedOperations = new AtomicInteger(expectedResults);
-        }
-
-        void run() {
-            if (expectedOperations.get() == 0) {
-                finish();
-                return;
-            }
-
-            logger.trace("mpercolate executing for shards {}", requestsByShard.keySet());
-            for (Map.Entry<ShardId, TransportShardMultiPercolateAction.Request> entry : requestsByShard.entrySet()) {
-                final ShardId shardId = entry.getKey();
-                TransportShardMultiPercolateAction.Request shardRequest = entry.getValue();
-                shardMultiPercolateAction.execute(shardRequest, new ActionListener<TransportShardMultiPercolateAction.Response>() {
-
+    private void innerDoExecute(MultiPercolateRequest request, ActionListener<MultiPercolateResponse> listener, Map<Integer, BytesReference> getResponseSources, Map<Integer, MultiPercolateResponse.Item> preFailures) {
+        try {
+            MultiSearchRequest multiSearchRequest = createMultiSearchRequest(request, getResponseSources, preFailures);
+            if (multiSearchRequest.requests().isEmpty()) {
+                // we may failed to turn all percolate requests into search requests,
+                // in that case just return the response...
+                listener.onResponse(
+                        createMultiPercolateResponse(new MultiSearchResponse(new MultiSearchResponse.Item[0]), request, preFailures)
+                );
+            } else {
+                client.multiSearch(multiSearchRequest, new ActionListener<MultiSearchResponse>() {
                     @Override
-                    public void onResponse(TransportShardMultiPercolateAction.Response response) {
-                        onShardResponse(shardId, response);
+                    public void onResponse(MultiSearchResponse response) {
+                        try {
+                            listener.onResponse(createMultiPercolateResponse(response, request, preFailures));
+                        } catch (Exception e) {
+                            onFailure(e);
+                        }
                     }
 
                     @Override
                     public void onFailure(Throwable e) {
-                        onShardFailure(shardId, e);
+                        listener.onFailure(e);
                     }
-
                 });
             }
+        } catch (Exception e) {
+            listener.onFailure(e);
         }
+    }
 
-        @SuppressWarnings("unchecked")
-        void onShardResponse(ShardId shardId, TransportShardMultiPercolateAction.Response response) {
-            logger.trace("{} Percolate shard response", shardId);
+    private MultiSearchRequest createMultiSearchRequest(MultiPercolateRequest multiPercolateRequest, Map<Integer, BytesReference> getResponseSources, Map<Integer, MultiPercolateResponse.Item> preFailures) throws IOException {
+        MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
+        multiSearchRequest.indicesOptions(multiPercolateRequest.indicesOptions());
+
+        for (int i  = 0; i < multiPercolateRequest.requests().size(); i++) {
+            if (preFailures.keySet().contains(i)) {
+                continue;
+            }
+
+            PercolateRequest percolateRequest = multiPercolateRequest.requests().get(i);
+            BytesReference docSource = getResponseSources.get(i);
             try {
-                for (TransportShardMultiPercolateAction.Response.Item item : response.items()) {
-                    AtomicReferenceArray shardResults = responsesByItemAndShard.get(item.slot());
-                    if (shardResults == null) {
-                        assert false : "shardResults can't be null";
-                        continue;
-                    }
-
-                    if (item.failed()) {
-                        shardResults.set(shardId.id(), new BroadcastShardOperationFailedException(shardId, item.error()));
-                    } else {
-                        shardResults.set(shardId.id(), item.response());
-                    }
-
-                    assert expectedOperationsPerItem.get(item.slot()).get() >= 1 : "slot[" + item.slot() + "] can't be lower than one";
-                    if (expectedOperationsPerItem.get(item.slot()).decrementAndGet() == 0) {
-                        // Failure won't bubble up, since we fail the whole request now via the catch clause below,
-                        // so expectedOperationsPerItem will not be decremented twice.
-                        reduce(item.slot());
-                    }
-                }
-            } catch (Throwable e) {
-                logger.error("{} Percolate original reduce error", e, shardId);
-                finalListener.onFailure(e);
+                SearchRequest searchRequest = TransportPercolateAction.createSearchRequest(
+                        percolateRequest, docSource, queryRegistry, aggParsers, parseFieldMatcher
+                );
+                multiSearchRequest.add(searchRequest);
+            } catch (Exception e) {
+                preFailures.put(i, new MultiPercolateResponse.Item(e));
             }
         }
 
-        @SuppressWarnings("unchecked")
-        void onShardFailure(ShardId shardId, Throwable e) {
-            logger.debug("{} Shard multi percolate failure", e, shardId);
-            try {
-                IntArrayList slots = shardToSlots.get(shardId);
-                for (int i = 0; i < slots.size(); i++) {
-                    int slot = slots.get(i);
-                    AtomicReferenceArray shardResults = responsesByItemAndShard.get(slot);
-                    if (shardResults == null) {
-                        continue;
-                    }
+        return multiSearchRequest;
+    }
 
-                    shardResults.set(shardId.id(), new BroadcastShardOperationFailedException(shardId, e));
-                    assert expectedOperationsPerItem.get(slot).get() >= 1 : "slot[" + slot + "] can't be lower than one. Caused by: " + e.getMessage();
-                    if (expectedOperationsPerItem.get(slot).decrementAndGet() == 0) {
-                        reduce(slot);
-                    }
-                }
-            } catch (Throwable t) {
-                logger.error("{} Percolate original reduce error, original error {}", t, shardId, e);
-                finalListener.onFailure(t);
-            }
-        }
-
-        void reduce(int slot) {
-            AtomicReferenceArray shardResponses = responsesByItemAndShard.get(slot);
-            PercolateResponse reducedResponse = TransportPercolateAction.reduce((PercolateRequest) percolateRequests.get(slot), shardResponses, percolatorService);
-            reducedResponses.set(slot, reducedResponse);
-            assert expectedOperations.get() >= 1 : "slot[" + slot + "] expected options should be >= 1 but is " + expectedOperations.get();
-            if (expectedOperations.decrementAndGet() == 0) {
-                finish();
-            }
-        }
-
-        void finish() {
-            MultiPercolateResponse.Item[] finalResponse = new MultiPercolateResponse.Item[reducedResponses.length()];
-            for (int slot = 0; slot < reducedResponses.length(); slot++) {
-                Object element = reducedResponses.get(slot);
-                assert element != null : "Element[" + slot + "] shouldn't be null";
-                if (element instanceof PercolateResponse) {
-                    finalResponse[slot] = new MultiPercolateResponse.Item((PercolateResponse) element);
-                } else if (element instanceof Throwable) {
-                    finalResponse[slot] = new MultiPercolateResponse.Item((Throwable)element);
-                } else if (element instanceof MultiGetResponse.Failure) {
-                    finalResponse[slot] = new MultiPercolateResponse.Item(((MultiGetResponse.Failure)element).getFailure());
+    private MultiPercolateResponse createMultiPercolateResponse(MultiSearchResponse multiSearchResponse, MultiPercolateRequest request, Map<Integer, MultiPercolateResponse.Item> preFailures) {
+        int searchResponseIndex = 0;
+        MultiPercolateResponse.Item[] percolateItems = new MultiPercolateResponse.Item[request.requests().size()];
+        for (int i = 0; i < percolateItems.length; i++) {
+            if (preFailures.keySet().contains(i)) {
+                percolateItems[i] = preFailures.get(i);
+            } else {
+                MultiSearchResponse.Item searchItem = multiSearchResponse.getResponses()[searchResponseIndex++];
+                if (searchItem.isFailure()) {
+                    percolateItems[i] = new MultiPercolateResponse.Item(searchItem.getFailure());
+                } else {
+                    PercolateRequest percolateRequest = request.requests().get(i);
+                    percolateItems[i] = new MultiPercolateResponse.Item(TransportPercolateAction.createPercolateResponse(searchItem.getResponse(), percolateRequest.onlyCount()));
                 }
             }
-            finalListener.onResponse(new MultiPercolateResponse(finalResponse));
         }
-
+        return new MultiPercolateResponse(percolateItems);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java b/core/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java
index 2a8f1a4ed24..b23ef04021e 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java
@@ -18,71 +18,74 @@
  */
 package org.elasticsearch.action.percolate;
 
-import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.get.TransportGetAction;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.DefaultShardOperationFailedException;
-import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
-import org.elasticsearch.action.support.broadcast.TransportBroadcastAction;
-import org.elasticsearch.cluster.ClusterService;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.block.ClusterBlockException;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
-import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.engine.DocumentMissingException;
-import org.elasticsearch.percolator.PercolateException;
-import org.elasticsearch.percolator.PercolatorService;
-import org.elasticsearch.tasks.Task;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.PercolatorQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.TemplateQueryParser;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.rest.action.support.RestActions;
+import org.elasticsearch.script.Template;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicReferenceArray;
+import java.util.Arrays;
 
-/**
- *
- */
-public class TransportPercolateAction extends TransportBroadcastAction<PercolateRequest, PercolateResponse, PercolateShardRequest, PercolateShardResponse> {
+public class TransportPercolateAction extends HandledTransportAction<PercolateRequest, PercolateResponse> {
 
-    private final PercolatorService percolatorService;
-    private final TransportGetAction getAction;
+    private final Client client;
+    private final ParseFieldMatcher parseFieldMatcher;
+    private final IndicesQueriesRegistry queryRegistry;
+    private final AggregatorParsers aggParsers;
 
     @Inject
-    public TransportPercolateAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
-                                    TransportService transportService, PercolatorService percolatorService,
-                                    TransportGetAction getAction, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, PercolateAction.NAME, threadPool, clusterService, transportService, actionFilters,
-                indexNameExpressionResolver, PercolateRequest::new, PercolateShardRequest::new, ThreadPool.Names.PERCOLATE);
-        this.percolatorService = percolatorService;
-        this.getAction = getAction;
+    public TransportPercolateAction(Settings settings, ThreadPool threadPool, TransportService transportService,
+                                    ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                    Client client, IndicesQueriesRegistry indicesQueriesRegistry, AggregatorParsers aggParsers) {
+        super(settings, PercolateAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, PercolateRequest::new);
+        this.client = client;
+        this.aggParsers = aggParsers;
+        this.parseFieldMatcher = new ParseFieldMatcher(settings);
+        this.queryRegistry = indicesQueriesRegistry;
     }
 
     @Override
-    protected void doExecute(Task task, final PercolateRequest request, final ActionListener<PercolateResponse> listener) {
-        request.startTime = System.currentTimeMillis();
+    protected void doExecute(PercolateRequest request, ActionListener<PercolateResponse> listener) {
         if (request.getRequest() != null) {
-            getAction.execute(request.getRequest(), new ActionListener<GetResponse>() {
+            client.get(request.getRequest(), new ActionListener<GetResponse>() {
                 @Override
                 public void onResponse(GetResponse getResponse) {
-                    if (!getResponse.isExists()) {
-                        onFailure(new DocumentMissingException(null, request.getRequest().type(), request.getRequest().id()));
-                        return;
+                    if (getResponse.isExists()) {
+                        innerDoExecute(request, getResponse.getSourceAsBytesRef(), listener);
+                    } else {
+                        onFailure(new ResourceNotFoundException("percolate document [{}/{}/{}] doesn't exist", request.getRequest().index(), request.getRequest().type(), request.getRequest().id()));
                     }
-
-                    BytesReference docSource = getResponse.getSourceAsBytesRef();
-                    TransportPercolateAction.super.doExecute(task, new PercolateRequest(request, docSource), listener);
                 }
 
                 @Override
@@ -91,99 +94,153 @@ public class TransportPercolateAction extends TransportBroadcastAction<Percolate
                 }
             });
         } else {
-            super.doExecute(task, request, listener);
+            innerDoExecute(request, null, listener);
         }
     }
 
-    @Override
-    protected ClusterBlockException checkGlobalBlock(ClusterState state, PercolateRequest request) {
-        return state.blocks().globalBlockedException(ClusterBlockLevel.READ);
-    }
-
-    @Override
-    protected ClusterBlockException checkRequestBlock(ClusterState state, PercolateRequest request, String[] concreteIndices) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.READ, concreteIndices);
-    }
-
-    @Override
-    protected PercolateResponse newResponse(PercolateRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
-        return reduce(request, shardsResponses, percolatorService);
-    }
-
-    public static PercolateResponse reduce(PercolateRequest request, AtomicReferenceArray shardsResponses, PercolatorService percolatorService) {
-        int successfulShards = 0;
-        int failedShards = 0;
-
-        List<PercolateShardResponse> shardResults = null;
-        List<ShardOperationFailedException> shardFailures = null;
-
-        boolean onlyCount = false;
-        for (int i = 0; i < shardsResponses.length(); i++) {
-            Object shardResponse = shardsResponses.get(i);
-            if (shardResponse == null) {
-                // simply ignore non active shards
-            } else if (shardResponse instanceof BroadcastShardOperationFailedException) {
-                failedShards++;
-                if (shardFailures == null) {
-                    shardFailures = new ArrayList<>();
-                }
-                shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
-            } else {
-                PercolateShardResponse percolateShardResponse = (PercolateShardResponse) shardResponse;
-                successfulShards++;
-                if (!percolateShardResponse.isEmpty()) {
-                    if (shardResults == null) {
-                        onlyCount = percolateShardResponse.onlyCount();
-                        shardResults = new ArrayList<>();
-                    }
-                    shardResults.add(percolateShardResponse);
-                }
-            }
-        }
-
-        if (shardResults == null) {
-            long tookInMillis = Math.max(1, System.currentTimeMillis() - request.startTime);
-            PercolateResponse.Match[] matches = request.onlyCount() ? null : PercolateResponse.EMPTY;
-            return new PercolateResponse(shardsResponses.length(), successfulShards, failedShards, shardFailures, tookInMillis, matches);
-        } else {
-            PercolatorService.ReduceResult result = null;
-            try {
-                result = percolatorService.reduce(onlyCount, shardResults);
-            } catch (IOException e) {
-                throw new ElasticsearchException("error during reduce phase", e);
-            }
-            long tookInMillis =  Math.max(1, System.currentTimeMillis() - request.startTime);
-            return new PercolateResponse(
-                    shardsResponses.length(), successfulShards, failedShards, shardFailures,
-                    result.matches(), result.count(), tookInMillis, result.reducedAggregations()
-            );
-        }
-    }
-
-    @Override
-    protected PercolateShardRequest newShardRequest(int numShards, ShardRouting shard, PercolateRequest request) {
-        return new PercolateShardRequest(shard.shardId(), numShards, request);
-    }
-
-    @Override
-    protected PercolateShardResponse newShardResponse() {
-        return new PercolateShardResponse();
-    }
-
-    @Override
-    protected GroupShardsIterator shards(ClusterState clusterState, PercolateRequest request, String[] concreteIndices) {
-        Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, request.routing(), request.indices());
-        return clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, request.preference());
-    }
-
-    @Override
-    protected PercolateShardResponse shardOperation(PercolateShardRequest request) {
+    private void innerDoExecute(PercolateRequest request, BytesReference docSource, ActionListener<PercolateResponse> listener) {
+        SearchRequest searchRequest;
         try {
-            return percolatorService.percolate(request);
-        } catch (Throwable e) {
-            logger.trace("{} failed to percolate", e, request.shardId());
-            throw new PercolateException(request.shardId(), "failed to percolate", e);
+            searchRequest = createSearchRequest(request, docSource, queryRegistry, aggParsers, parseFieldMatcher);
+        } catch (IOException e) {
+            listener.onFailure(e);
+            return;
         }
+        client.search(searchRequest, new ActionListener<SearchResponse>() {
+            @Override
+            public void onResponse(SearchResponse searchResponse) {
+                try {
+                    listener.onResponse(createPercolateResponse(searchResponse, request.onlyCount()));
+                } catch (Exception e) {
+                    onFailure(e);
+                }
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+                listener.onFailure(e);
+            }
+        });
+    }
+
+    public static SearchRequest createSearchRequest(PercolateRequest percolateRequest, BytesReference documentSource, IndicesQueriesRegistry queryRegistry, AggregatorParsers aggParsers, ParseFieldMatcher parseFieldMatcher) throws IOException {
+        SearchRequest searchRequest = new SearchRequest();
+        if (percolateRequest.indices() != null) {
+            searchRequest.indices(percolateRequest.indices());
+        }
+        searchRequest.indicesOptions(percolateRequest.indicesOptions());
+        searchRequest.routing(percolateRequest.routing());
+        searchRequest.preference(percolateRequest.preference());
+
+        BytesReference querySource = null;
+        XContentBuilder searchSource = XContentFactory.jsonBuilder().startObject();
+        if (percolateRequest.source() != null && percolateRequest.source().length() > 0) {
+            try (XContentParser parser = XContentHelper.createParser(percolateRequest.source())) {
+                String currentFieldName = null;
+                XContentParser.Token token = parser.nextToken();
+                if (token != XContentParser.Token.START_OBJECT) {
+                    throw new IllegalArgumentException("Unknown token [" + token+ "]");
+                }
+
+                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    if (token == XContentParser.Token.FIELD_NAME) {
+                        currentFieldName = parser.currentName();
+                    } else if (token == XContentParser.Token.START_OBJECT) {
+                        if ("doc".equals(currentFieldName)) {
+                            XContentBuilder builder = XContentFactory.jsonBuilder();
+                            builder.copyCurrentStructure(parser);
+                            builder.flush();
+                            documentSource = builder.bytes();
+                        } else if ("query".equals(currentFieldName) || "filter".equals(currentFieldName)) {
+                            XContentBuilder builder = XContentFactory.jsonBuilder();
+                            builder.copyCurrentStructure(parser);
+                            builder.flush();
+                            querySource = builder.bytes();
+                        } else if ("sort".equals(currentFieldName)) {
+                            searchSource.field("sort");
+                            searchSource.copyCurrentStructure(parser);
+                        } else if ("aggregations".equals(currentFieldName)) {
+                            searchSource.field("aggregations");
+                            searchSource.copyCurrentStructure(parser);
+                        } else if ("highlight".equals(currentFieldName)) {
+                            searchSource.field("highlight");
+                            searchSource.copyCurrentStructure(parser);
+                        } else {
+                            throw new IllegalArgumentException("Unknown field [" + currentFieldName+ "]");
+                        }
+                    } else if (token == XContentParser.Token.START_ARRAY) {
+                        if ("sort".equals(currentFieldName)) {
+                            searchSource.field("sort");
+                            searchSource.copyCurrentStructure(parser);
+                        } else {
+                            throw new IllegalArgumentException("Unknown field [" + currentFieldName+ "]");
+                        }
+                    } else if (token.isValue()) {
+                        if ("size".equals(currentFieldName)) {
+                            searchSource.field("size", parser.intValue());
+                        } else if ("sort".equals(currentFieldName)) {
+                            searchSource.field("sort", parser.text());
+                        } else if ("track_scores".equals(currentFieldName) || "trackScores".equals(currentFieldName)) {
+                            searchSource.field("track_scores", parser.booleanValue());
+                        } else {
+                            throw new IllegalArgumentException("Unknown field [" + currentFieldName+ "]");
+                        }
+                    } else {
+                        throw new IllegalArgumentException("Unknown token [" + token + "]");
+                    }
+                }
+            }
+        }
+
+        if (percolateRequest.onlyCount()) {
+            searchSource.field("size", 0);
+        }
+
+        PercolatorQueryBuilder percolatorQueryBuilder = new PercolatorQueryBuilder(percolateRequest.documentType(), documentSource);
+        if (querySource != null) {
+            QueryParseContext queryParseContext = new QueryParseContext(queryRegistry);
+            queryParseContext.reset(XContentHelper.createParser(querySource));
+            queryParseContext.parseFieldMatcher(parseFieldMatcher);
+            QueryBuilder<?> queryBuilder = queryParseContext.parseInnerQueryBuilder();
+            BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+            boolQueryBuilder.must(queryBuilder);
+            boolQueryBuilder.filter(percolatorQueryBuilder);
+            searchSource.field("query", boolQueryBuilder);
+        } else {
+            searchSource.field("query", percolatorQueryBuilder);
+        }
+
+        searchSource.endObject();
+        searchSource.flush();
+        BytesReference source = searchSource.bytes();
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        QueryParseContext context = new QueryParseContext(queryRegistry);
+        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(source)) {
+            context.reset(parser);
+            context.parseFieldMatcher(parseFieldMatcher);
+            searchSourceBuilder.parseXContent(parser, context, aggParsers, null);
+            searchRequest.source(searchSourceBuilder);
+            return searchRequest;
+        }
+    }
+
+    public static PercolateResponse createPercolateResponse(SearchResponse searchResponse, boolean onlyCount) {
+        SearchHits hits = searchResponse.getHits();
+        PercolateResponse.Match[] matches;
+        if (onlyCount) {
+            matches = null;
+        } else {
+            matches = new PercolateResponse.Match[hits.getHits().length];
+            for (int i = 0; i < hits.getHits().length; i++) {
+                SearchHit hit = hits.getHits()[i];
+                matches[i] = new PercolateResponse.Match(new Text(hit.getIndex()), new Text(hit.getId()), hit.getScore(), hit.getHighlightFields());
+            }
+        }
+
+        return new PercolateResponse(
+            searchResponse.getTotalShards(), searchResponse.getSuccessfulShards(), searchResponse.getFailedShards(),
+            Arrays.asList(searchResponse.getShardFailures()), matches, hits.getTotalHits(), searchResponse.getTookInMillis(), (InternalAggregations) searchResponse.getAggregations()
+        );
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/TransportShardMultiPercolateAction.java b/core/src/main/java/org/elasticsearch/action/percolate/TransportShardMultiPercolateAction.java
deleted file mode 100644
index 0732d4d4066..00000000000
--- a/core/src/main/java/org/elasticsearch/action/percolate/TransportShardMultiPercolateAction.java
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.percolate;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.IndicesRequest;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.TransportActions;
-import org.elasticsearch.action.support.single.shard.SingleShardRequest;
-import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
-import org.elasticsearch.cluster.ClusterService;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.ShardIterator;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.percolator.PercolatorService;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/**
- */
-public class TransportShardMultiPercolateAction extends TransportSingleShardAction<TransportShardMultiPercolateAction.Request, TransportShardMultiPercolateAction.Response> {
-
-    private final PercolatorService percolatorService;
-
-    private static final String ACTION_NAME = MultiPercolateAction.NAME + "[shard]";
-
-    @Inject
-    public TransportShardMultiPercolateAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
-                                              TransportService transportService, PercolatorService percolatorService,
-                                              ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, ACTION_NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
-                Request::new, ThreadPool.Names.PERCOLATE);
-        this.percolatorService = percolatorService;
-    }
-
-    @Override
-    protected boolean isSubAction() {
-        return true;
-    }
-
-    @Override
-    protected Response newResponse() {
-        return new Response();
-    }
-
-    @Override
-    protected boolean resolveIndex(Request request) {
-        return false;
-    }
-
-    @Override
-    protected ShardIterator shards(ClusterState state, InternalRequest request) {
-        return clusterService.operationRouting().getShards(
-                state, request.concreteIndex(), request.request().shardId(), request.request().preference
-        );
-    }
-
-    @Override
-    protected Response shardOperation(Request request, ShardId shardId) {
-        // TODO: Look into combining the shard req's docs into one in memory index.
-        Response response = new Response();
-        response.items = new ArrayList<>(request.items.size());
-        for (Request.Item item : request.items) {
-            Response.Item responseItem;
-            int slot = item.slot;
-            try {
-                responseItem = new Response.Item(slot, percolatorService.percolate(item.request));
-            } catch (Throwable t) {
-                if (TransportActions.isShardNotAvailableException(t)) {
-                    throw (ElasticsearchException) t;
-                } else {
-                    logger.debug("{} failed to multi percolate", t, request.shardId());
-                    responseItem = new Response.Item(slot, t);
-                }
-            }
-            response.items.add(responseItem);
-        }
-        return response;
-    }
-
-
-    public static class Request extends SingleShardRequest<Request> implements IndicesRequest {
-
-        private int shardId;
-        private String preference;
-        private List<Item> items;
-
-        public Request() {
-        }
-
-        Request(String concreteIndex, int shardId, String preference) {
-            super(concreteIndex);
-            this.shardId = shardId;
-            this.preference = preference;
-            this.items = new ArrayList<>();
-        }
-
-        @Override
-        public ActionRequestValidationException validate() {
-            return super.validateNonNullIndex();
-        }
-
-        @Override
-        public String[] indices() {
-            List<String> indices = new ArrayList<>();
-            for (Item item : items) {
-                Collections.addAll(indices, item.request.indices());
-            }
-            return indices.toArray(new String[indices.size()]);
-        }
-
-        public int shardId() {
-            return shardId;
-        }
-
-        public void add(Item item) {
-            items.add(item);
-        }
-
-        public List<Item> items() {
-            return items;
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            shardId = in.readVInt();
-            preference = in.readOptionalString();
-            int size = in.readVInt();
-            items = new ArrayList<>(size);
-            for (int i = 0; i < size; i++) {
-                int slot = in.readVInt();
-                PercolateShardRequest shardRequest = new PercolateShardRequest();
-                shardRequest.readFrom(in);
-                Item item = new Item(slot, shardRequest);
-                items.add(item);
-            }
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeVInt(shardId);
-            out.writeOptionalString(preference);
-            out.writeVInt(items.size());
-            for (Item item : items) {
-                out.writeVInt(item.slot);
-                item.request.writeTo(out);
-            }
-        }
-
-        static class Item {
-
-            private final int slot;
-            private final PercolateShardRequest request;
-
-            public Item(int slot, PercolateShardRequest request) {
-                this.slot = slot;
-                this.request = request;
-            }
-
-            public int slot() {
-                return slot;
-            }
-
-            public PercolateShardRequest request() {
-                return request;
-            }
-
-        }
-
-    }
-
-    public static class Response extends ActionResponse {
-
-        private List<Item> items;
-
-        public List<Item> items() {
-            return items;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeVInt(items.size());
-            for (Item item : items) {
-                out.writeVInt(item.slot);
-                if (item.response != null) {
-                    out.writeBoolean(true);
-                    item.response.writeTo(out);
-                } else {
-                    out.writeBoolean(false);
-                    out.writeThrowable(item.error);
-                }
-            }
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            int size = in.readVInt();
-            items = new ArrayList<>(size);
-            for (int i = 0; i < size; i++) {
-                int slot = in.readVInt();
-                if (in.readBoolean()) {
-                    PercolateShardResponse shardResponse = new PercolateShardResponse();
-                    shardResponse.readFrom(in);
-                    items.add(new Item(slot, shardResponse));
-                } else {
-                    items.add(new Item(slot, in.readThrowable()));
-                }
-            }
-        }
-
-        public static class Item {
-
-            private final int slot;
-            private final PercolateShardResponse response;
-            private final Throwable error;
-
-            public Item(Integer slot, PercolateShardResponse response) {
-                this.slot = slot;
-                this.response = response;
-                this.error = null;
-            }
-
-            public Item(Integer slot, Throwable error) {
-                this.slot = slot;
-                this.error = error;
-                this.response = null;
-            }
-
-            public int slot() {
-                return slot;
-            }
-
-            public PercolateShardResponse response() {
-                return response;
-            }
-
-            public Throwable error() {
-                return error;
-            }
-
-            public boolean failed() {
-                return error != null;
-            }
-        }
-
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index 830a54778e1..732e9098ee7 100644
--- a/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -25,7 +25,6 @@ import org.apache.lucene.search.TopDocs;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.NoShardAvailableActionException;
 import org.elasticsearch.action.support.TransportActions;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -34,6 +33,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
@@ -96,7 +96,7 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
         // TODO: I think startTime() should become part of ActionRequest and that should be used both for index name
         // date math expressions and $now in scripts. This way all apis will deal with now in the same way instead
         // of just for the _search api
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, request.indicesOptions(),
+        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request.indicesOptions(),
             startTime(), request.indices());
 
         for (String index : concreteIndices) {
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java
index 56d0fedd40c..f7cb72b22e9 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java
@@ -21,9 +21,9 @@ package org.elasticsearch.action.search;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.action.SearchTransportService;
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
index f2dcefa7554..c5f320f1b33 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
@@ -23,9 +23,9 @@ import com.carrotsearch.hppc.IntArrayList;
 import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.SearchShardTarget;
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java
index dcbf9b5091f..1b338847762 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java
@@ -21,9 +21,9 @@ package org.elasticsearch.action.search;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.search.action.SearchTransportService;
 import org.elasticsearch.search.controller.SearchPhaseController;
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
index e15b9da8acb..3feb40411f9 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
@@ -23,9 +23,9 @@ import com.carrotsearch.hppc.IntArrayList;
 import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.SearchShardTarget;
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 9d3c200ed98..e87d393bf05 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -295,6 +295,13 @@ public class SearchRequest extends ActionRequest<SearchRequest> implements Indic
         return this.requestCache;
     }
 
+    /**
+     * @return true if the request only has suggest
+     */
+    public boolean isSuggestOnly() {
+        return source != null && source.isSuggestOnly();
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index ffca87de22c..931df24a256 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -393,9 +393,7 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
     }
 
     /**
-     * Delegates to
-     * {@link org.elasticsearch.search.suggest.SuggestBuilder#addSuggestion(org.elasticsearch.search.suggest.SuggestBuilder.SuggestionBuilder)}
-     * .
+     * Delegates to {@link SearchSourceBuilder#suggest(SuggestBuilder)}
      */
     public SearchRequestBuilder suggest(SuggestBuilder suggestBuilder) {
         sourceBuilder().suggest(suggestBuilder);
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java
index b5b95dc5cbe..4e0ee3ff5e5 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java
@@ -21,9 +21,9 @@ package org.elasticsearch.action.search;
 
 import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.action.SearchTransportService;
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
index 864f17eee2c..8e822302d2f 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
@@ -22,9 +22,9 @@ package org.elasticsearch.action.search;
 import com.carrotsearch.hppc.IntArrayList;
 import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.action.SearchTransportService;
diff --git a/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java b/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
index 0139186562c..2a01eb4e1c6 100644
--- a/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
+++ b/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
@@ -32,8 +32,6 @@ import org.elasticsearch.search.SearchShardTarget;
 
 import java.io.IOException;
 
-import static org.elasticsearch.search.SearchShardTarget.readSearchShardTarget;
-
 /**
  * Represents a failure to search on a specific shard.
  */
@@ -106,7 +104,7 @@ public class ShardSearchFailure implements ShardOperationFailedException {
     @Override
     public int shardId() {
         if (shardTarget != null) {
-            return shardTarget.shardId();
+            return shardTarget.shardId().id();
         }
         return -1;
     }
@@ -133,7 +131,7 @@ public class ShardSearchFailure implements ShardOperationFailedException {
     @Override
     public void readFrom(StreamInput in) throws IOException {
         if (in.readBoolean()) {
-            shardTarget = readSearchShardTarget(in);
+            shardTarget = new SearchShardTarget(in);
         }
         reason = in.readString();
         status = RestStatus.readFrom(in);
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
index 95f0796ba4f..fb021fdd9f9 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
@@ -22,11 +22,11 @@ package org.elasticsearch.action.search;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.CountDown;
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
index 43754427598..c0428cd531f 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
@@ -22,10 +22,10 @@ package org.elasticsearch.action.search;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index e87fa2a345a..5b20a05bad5 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -22,9 +22,9 @@ package org.elasticsearch.action.search;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -38,6 +38,7 @@ import java.util.Map;
 import java.util.Set;
 
 import static org.elasticsearch.action.search.SearchType.QUERY_AND_FETCH;
+import static org.elasticsearch.action.search.SearchType.QUERY_THEN_FETCH;
 
 /**
  *
@@ -64,7 +65,7 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
         // optimize search type for cases where there is only one shard group to search on
         try {
             ClusterState clusterState = clusterService.state();
-            String[] concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, searchRequest);
+            String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, searchRequest);
             Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState,
                     searchRequest.routing(), searchRequest.indices());
             int shardCount = clusterService.operationRouting().searchShardsCount(clusterState, concreteIndices, routingMap);
@@ -72,6 +73,17 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
                 // if we only have one group, then we always want Q_A_F, no need for DFS, and no need to do THEN since we hit one shard
                 searchRequest.searchType(QUERY_AND_FETCH);
             }
+            if (searchRequest.isSuggestOnly()) {
+                // disable request cache if we have only suggest
+                searchRequest.requestCache(false);
+                switch (searchRequest.searchType()) {
+                    case DFS_QUERY_AND_FETCH:
+                    case DFS_QUERY_THEN_FETCH:
+                        // convert to Q_T_F if we have only suggest
+                        searchRequest.searchType(QUERY_THEN_FETCH);
+                        break;
+                }
+            }
         } catch (IndexNotFoundException | IndexClosedException e) {
             // ignore these failures, we will notify the search response if its really the case from the actual action
         } catch (Exception e) {
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java
index 03009cc0102..3bcadda1725 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java
@@ -22,8 +22,8 @@ package org.elasticsearch.action.search;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.search.action.SearchTransportService;
diff --git a/core/src/main/java/org/elasticsearch/action/suggest/ShardSuggestRequest.java b/core/src/main/java/org/elasticsearch/action/suggest/ShardSuggestRequest.java
deleted file mode 100644
index 80facf74878..00000000000
--- a/core/src/main/java/org/elasticsearch/action/suggest/ShardSuggestRequest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.suggest;
-
-import org.elasticsearch.action.support.broadcast.BroadcastShardRequest;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.index.shard.ShardId;
-
-import java.io.IOException;
-
-/**
- * Internal suggest request executed directly against a specific index shard.
- */
-public final class ShardSuggestRequest extends BroadcastShardRequest {
-
-    private BytesReference suggestSource;
-
-    public ShardSuggestRequest() {
-    }
-
-    ShardSuggestRequest(ShardId shardId, SuggestRequest request) {
-        super(shardId, request);
-        this.suggestSource = request.suggest();
-    }
-
-    public BytesReference suggest() {
-        return suggestSource;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        suggestSource = in.readBytesReference();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeBytesReference(suggestSource);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/action/suggest/ShardSuggestResponse.java b/core/src/main/java/org/elasticsearch/action/suggest/ShardSuggestResponse.java
deleted file mode 100644
index bca29800bd1..00000000000
--- a/core/src/main/java/org/elasticsearch/action/suggest/ShardSuggestResponse.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.suggest;
-
-import org.elasticsearch.action.support.broadcast.BroadcastShardResponse;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.search.suggest.Suggest;
-
-import java.io.IOException;
-
-/**
- * Internal suggest response of a shard suggest request executed directly against a specific shard.
- */
-class ShardSuggestResponse extends BroadcastShardResponse {
-
-    private final Suggest suggest;
-
-    ShardSuggestResponse() {
-        this.suggest = new Suggest();
-    }
-
-    ShardSuggestResponse(ShardId shardId, Suggest suggest) {
-        super(shardId);
-        this.suggest = suggest;
-    }
-
-    public Suggest getSuggest() {
-        return this.suggest;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        suggest.readFrom(in);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        suggest.writeTo(out);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/action/suggest/SuggestAction.java b/core/src/main/java/org/elasticsearch/action/suggest/SuggestAction.java
deleted file mode 100644
index 88d2a92f331..00000000000
--- a/core/src/main/java/org/elasticsearch/action/suggest/SuggestAction.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.suggest;
-
-import org.elasticsearch.action.Action;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.search.suggest.Suggest;
-
-/**
- */
-public class SuggestAction extends Action<SuggestRequest, SuggestResponse, SuggestRequestBuilder> {
-
-    public static final SuggestAction INSTANCE = new SuggestAction();
-    public static final String NAME = "indices:data/read/suggest";
-
-    private SuggestAction() {
-        super(NAME);
-    }
-
-    @Override
-    public SuggestResponse newResponse() {
-        return new SuggestResponse(new Suggest());
-    }
-
-    @Override
-    public SuggestRequestBuilder newRequestBuilder(ElasticsearchClient client) {
-        return new SuggestRequestBuilder(client, this);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequest.java b/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequest.java
deleted file mode 100644
index 0d1c4932d48..00000000000
--- a/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequest.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.suggest;
-
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.support.broadcast.BroadcastRequest;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.search.suggest.SuggestBuilder;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-/**
- * A request to get suggestions for corrections of phrases. Best created with
- * {@link org.elasticsearch.client.Requests#suggestRequest(String...)}.
- * <p>
- * The request requires the suggest query source to be set either using
- * {@link #suggest(org.elasticsearch.common.bytes.BytesReference)} / {@link #suggest(org.elasticsearch.common.bytes.BytesReference)}
- * or by using {@link #suggest(org.elasticsearch.search.suggest.SuggestBuilder)}
- * (Best created using the {link @org.elasticsearch.search.suggest.SuggestBuilders)}).
- *
- * @see SuggestResponse
- * @see org.elasticsearch.client.Client#suggest(SuggestRequest)
- * @see org.elasticsearch.client.Requests#suggestRequest(String...)
- * @see org.elasticsearch.search.suggest.SuggestBuilders
- */
-public final class SuggestRequest extends BroadcastRequest<SuggestRequest> {
-
-    @Nullable
-    private String routing;
-
-    @Nullable
-    private String preference;
-
-    private BytesReference suggestSource;
-
-    public SuggestRequest() {
-    }
-
-    /**
-     * Constructs a new suggest request against the provided indices. No indices provided means it will
-     * run against all indices.
-     */
-    public SuggestRequest(String... indices) {
-        super(indices);
-    }
-
-    @Override
-    public ActionRequestValidationException validate() {
-        ActionRequestValidationException validationException = super.validate();
-        return validationException;
-    }
-
-    /**
-     * The Phrase to get correction suggestions for 
-     */
-    public BytesReference suggest() {
-        return suggestSource;
-    }
-    
-    /**
-     * set a new source for the suggest query  
-     */
-    public SuggestRequest suggest(BytesReference suggestSource) {
-        this.suggestSource = suggestSource;
-        return this;
-    }
-
-    /**
-     * set a new source using a {@link org.elasticsearch.search.suggest.SuggestBuilder}
-     * for phrase and term suggestion lookup
-     */
-    public SuggestRequest suggest(SuggestBuilder suggestBuilder) {
-        return suggest(suggestBuilder.buildAsBytes(Requests.CONTENT_TYPE));
-    }
-
-    /**
-     * set a new source using a {@link org.elasticsearch.search.suggest.SuggestBuilder.SuggestionBuilder}
-     * for completion suggestion lookup
-     */
-    public SuggestRequest suggest(SuggestBuilder.SuggestionBuilder suggestionBuilder) {
-        return suggest(suggestionBuilder.buildAsBytes(Requests.CONTENT_TYPE));
-    }
-
-    public SuggestRequest suggest(String source) {
-        return suggest(new BytesArray(source));
-    }
-
-    /**
-     * A comma separated list of routing values to control the shards the search will be executed on.
-     */
-    public String routing() {
-        return this.routing;
-    }
-
-    /**
-     * A comma separated list of routing values to control the shards the search will be executed on.
-     */
-    public SuggestRequest routing(String routing) {
-        this.routing = routing;
-        return this;
-    }
-
-    /**
-     * The routing values to control the shards that the search will be executed on.
-     */
-    public SuggestRequest routing(String... routings) {
-        this.routing = Strings.arrayToCommaDelimitedString(routings);
-        return this;
-    }
-
-    public SuggestRequest preference(String preference) {
-        this.preference = preference;
-        return this;
-    }
-
-    public String preference() {
-        return this.preference;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        routing = in.readOptionalString();
-        preference = in.readOptionalString();
-        suggest(in.readBytesReference());
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeOptionalString(routing);
-        out.writeOptionalString(preference);
-        out.writeBytesReference(suggestSource);
-    }
-
-    @Override
-    public String toString() {
-        String sSource = "_na_";
-        try {
-            sSource = XContentHelper.convertToJson(suggestSource, false);
-        } catch (Exception e) {
-            // ignore
-        }
-        return "[" + Arrays.toString(indices) + "]" + ", suggestSource[" + sSource + "]";
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequestBuilder.java
deleted file mode 100644
index 06a2b00c648..00000000000
--- a/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequestBuilder.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.suggest;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.support.broadcast.BroadcastOperationRequestBuilder;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.search.suggest.SuggestBuilder;
-import org.elasticsearch.search.suggest.SuggestBuilder.SuggestionBuilder;
-
-import java.io.IOException;
-
-/**
- * A suggest action request builder.
- */
-public class SuggestRequestBuilder extends BroadcastOperationRequestBuilder<SuggestRequest, SuggestResponse, SuggestRequestBuilder> {
-
-    final SuggestBuilder suggest = new SuggestBuilder();
-
-    public SuggestRequestBuilder(ElasticsearchClient client, SuggestAction action) {
-        super(client, action, new SuggestRequest());
-    }
-
-    /**
-     * Add a definition for suggestions to the request
-     */
-    public <T> SuggestRequestBuilder addSuggestion(SuggestionBuilder<T> suggestion) {
-        suggest.addSuggestion(suggestion);
-        return this;
-    }
-
-    /**
-     * A comma separated list of routing values to control the shards the search will be executed on.
-     */
-    public SuggestRequestBuilder setRouting(String routing) {
-        request.routing(routing);
-        return this;
-    }
-
-    public SuggestRequestBuilder setSuggestText(String globalText) {
-        this.suggest.setText(globalText);
-        return this;
-    }
-
-    /**
-     * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * <tt>_local</tt> to prefer local shards, <tt>_primary</tt> to execute only on primary shards,
-     * _shards:x,y to operate on shards x &amp; y, or a custom value, which guarantees that the same order
-     * will be used across different requests.
-     */
-    public SuggestRequestBuilder setPreference(String preference) {
-        request.preference(preference);
-        return this;
-    }
-
-    /**
-     * The routing values to control the shards that the search will be executed on.
-     */
-    public SuggestRequestBuilder setRouting(String... routing) {
-        request.routing(routing);
-        return this;
-    }
-
-    @Override
-    protected SuggestRequest beforeExecute(SuggestRequest request) {
-        try {
-            XContentBuilder builder = XContentFactory.contentBuilder(Requests.CONTENT_TYPE);
-            suggest.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            request.suggest(builder.bytes());
-        } catch (IOException e) {
-            throw new ElasticsearchException("Unable to build suggestion request", e);
-        }
-        return request;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/action/suggest/SuggestResponse.java b/core/src/main/java/org/elasticsearch/action/suggest/SuggestResponse.java
deleted file mode 100644
index 445e804b5b5..00000000000
--- a/core/src/main/java/org/elasticsearch/action/suggest/SuggestResponse.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.suggest;
-
-import org.elasticsearch.action.ShardOperationFailedException;
-import org.elasticsearch.action.support.broadcast.BroadcastResponse;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.search.suggest.Suggest;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
-
-/**
- * The response of the suggest action.
- */
-public final class SuggestResponse extends BroadcastResponse {
-
-    private final Suggest suggest;
-
-    SuggestResponse(Suggest suggest) {
-        this.suggest = suggest;
-    }
-
-    SuggestResponse(Suggest suggest, int totalShards, int successfulShards, int failedShards, List<ShardOperationFailedException> shardFailures) {
-        super(totalShards, successfulShards, failedShards, shardFailures);
-        this.suggest = suggest;
-    }
-
-    /**
-     * The Suggestions of the phrase.
-     */
-    public Suggest getSuggest() {
-        return suggest;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        this.suggest.readFrom(in);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        this.suggest.writeTo(out);
-    }
-
-    @Override
-    public String toString() {
-        try {
-            XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
-            builder.startObject();
-            suggest.toXContent(builder, EMPTY_PARAMS);
-            builder.endObject();
-            return builder.string();
-        } catch (IOException e) {
-            return "{ \"error\" : \"" + e.getMessage() + "\"}";
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java b/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java
deleted file mode 100644
index 0ed98578557..00000000000
--- a/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.suggest;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.ShardOperationFailedException;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.DefaultShardOperationFailedException;
-import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
-import org.elasticsearch.action.support.broadcast.TransportBroadcastAction;
-import org.elasticsearch.cluster.ClusterService;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.block.ClusterBlockException;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
-import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.suggest.stats.ShardSuggestMetric;
-import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.search.suggest.Suggest;
-import org.elasticsearch.search.suggest.SuggestPhase;
-import org.elasticsearch.search.suggest.SuggestionSearchContext;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicReferenceArray;
-
-/**
- * Defines the transport of a suggestion request across the cluster
- */
-public class TransportSuggestAction extends TransportBroadcastAction<SuggestRequest, SuggestResponse, ShardSuggestRequest, ShardSuggestResponse> {
-
-    private final IndicesService indicesService;
-    private final SuggestPhase suggestPhase;
-
-    @Inject
-    public TransportSuggestAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService,
-                                  IndicesService indicesService, SuggestPhase suggestPhase, ActionFilters actionFilters,
-                                  IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, SuggestAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
-                SuggestRequest::new, ShardSuggestRequest::new, ThreadPool.Names.SUGGEST);
-        this.indicesService = indicesService;
-        this.suggestPhase = suggestPhase;
-    }
-
-    @Override
-    protected ShardSuggestRequest newShardRequest(int numShards, ShardRouting shard, SuggestRequest request) {
-        return new ShardSuggestRequest(shard.shardId(), request);
-    }
-
-    @Override
-    protected ShardSuggestResponse newShardResponse() {
-        return new ShardSuggestResponse();
-    }
-
-    @Override
-    protected GroupShardsIterator shards(ClusterState clusterState, SuggestRequest request, String[] concreteIndices) {
-        Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, request.routing(), request.indices());
-        return clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, request.preference());
-    }
-
-    @Override
-    protected ClusterBlockException checkGlobalBlock(ClusterState state, SuggestRequest request) {
-        return state.blocks().globalBlockedException(ClusterBlockLevel.READ);
-    }
-
-    @Override
-    protected ClusterBlockException checkRequestBlock(ClusterState state, SuggestRequest countRequest, String[] concreteIndices) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.READ, concreteIndices);
-    }
-
-    @Override
-    protected SuggestResponse newResponse(SuggestRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
-        int successfulShards = 0;
-        int failedShards = 0;
-
-        final Map<String, List<Suggest.Suggestion>> groupedSuggestions = new HashMap<>();
-
-        List<ShardOperationFailedException> shardFailures = null;
-        for (int i = 0; i < shardsResponses.length(); i++) {
-            Object shardResponse = shardsResponses.get(i);
-            if (shardResponse == null) {
-                // simply ignore non active shards
-            } else if (shardResponse instanceof BroadcastShardOperationFailedException) {
-                failedShards++;
-                if (shardFailures == null) {
-                    shardFailures = new ArrayList<>();
-                }
-                shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
-            } else {
-                Suggest suggest = ((ShardSuggestResponse) shardResponse).getSuggest();
-                Suggest.group(groupedSuggestions, suggest);
-                successfulShards++;
-            }
-        }
-
-        return new SuggestResponse(new Suggest(Suggest.reduce(groupedSuggestions)), shardsResponses.length(), successfulShards, failedShards, shardFailures);
-    }
-
-    @Override
-    protected ShardSuggestResponse shardOperation(ShardSuggestRequest request) {
-        IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
-        IndexShard indexShard = indexService.getShard(request.shardId().id());
-        ShardSuggestMetric suggestMetric = indexShard.getSuggestMetric();
-        suggestMetric.preSuggest();
-        long startTime = System.nanoTime();
-        XContentParser parser = null;
-        try (Engine.Searcher searcher = indexShard.acquireSearcher("suggest")) {
-            BytesReference suggest = request.suggest();
-            if (suggest != null && suggest.length() > 0) {
-                parser = XContentFactory.xContent(suggest).createParser(suggest);
-                if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
-                    throw new IllegalArgumentException("suggest content missing");
-                }
-                final SuggestionSearchContext context = suggestPhase.parseElement().parseInternal(parser, indexService.mapperService(),
-                        indexService.fieldData(), request.shardId().getIndexName(), request.shardId().id());
-                final Suggest result = suggestPhase.execute(context, searcher.searcher());
-                return new ShardSuggestResponse(request.shardId(), result);
-            }
-            return new ShardSuggestResponse(request.shardId(), new Suggest());
-        } catch (Throwable ex) {
-            throw new ElasticsearchException("failed to execute suggest", ex);
-        } finally {
-            if (parser != null) {
-                parser.close();
-            }
-            suggestMetric.postSuggest(System.nanoTime() - startTime);
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/action/suggest/package-info.java b/core/src/main/java/org/elasticsearch/action/suggest/package-info.java
deleted file mode 100644
index a2c0f48ea51..00000000000
--- a/core/src/main/java/org/elasticsearch/action/suggest/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Suggest action.
- */
-package org.elasticsearch.action.suggest;
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java b/core/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java
index d5574755346..339abcb22bc 100644
--- a/core/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java
+++ b/core/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.MapperService;
 
@@ -39,7 +40,8 @@ import java.util.List;
  */
 public final class AutoCreateIndex {
 
-    public static final Setting<AutoCreate> AUTO_CREATE_INDEX_SETTING = new Setting<>("action.auto_create_index", "true", AutoCreate::new, false, Setting.Scope.CLUSTER);
+    public static final Setting<AutoCreate> AUTO_CREATE_INDEX_SETTING =
+        new Setting<>("action.auto_create_index", "true", AutoCreate::new, Property.NodeScope);
 
     private final boolean dynamicMappingDisabled;
     private final IndexNameExpressionResolver resolver;
diff --git a/core/src/main/java/org/elasticsearch/action/support/DestructiveOperations.java b/core/src/main/java/org/elasticsearch/action/support/DestructiveOperations.java
index 5f2fb33e043..31fc1d06175 100644
--- a/core/src/main/java/org/elasticsearch/action/support/DestructiveOperations.java
+++ b/core/src/main/java/org/elasticsearch/action/support/DestructiveOperations.java
@@ -23,6 +23,7 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 /**
@@ -33,7 +34,8 @@ public final class DestructiveOperations extends AbstractComponent {
     /**
      * Setting which controls whether wildcard usage (*, prefix*, _all) is allowed.
      */
-    public static final Setting<Boolean> REQUIRES_NAME_SETTING = Setting.boolSetting("action.destructive_requires_name", false, true, Setting.Scope.CLUSTER);
+    public static final Setting<Boolean> REQUIRES_NAME_SETTING =
+        Setting.boolSetting("action.destructive_requires_name", false, Property.Dynamic, Property.NodeScope);
     private volatile boolean destructiveRequiresName;
 
     @Inject
diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
index e39fb0288ac..182d922fc39 100644
--- a/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.NoShardAvailableActionException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.TransportActions;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -33,6 +32,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
@@ -125,7 +125,7 @@ public abstract class TransportBroadcastAction<Request extends BroadcastRequest<
                 throw blockException;
             }
             // update to concrete indices
-            String[] concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, request);
+            String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request);
             blockException = checkRequestBlock(clusterState, request, concreteIndices);
             if (blockException != null) {
                 throw blockException;
diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
index 76ed3c67daa..3ed3e8ffb7a 100644
--- a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
@@ -33,7 +33,6 @@ import org.elasticsearch.action.support.TransportActions;
 import org.elasticsearch.action.support.broadcast.BroadcastRequest;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -41,6 +40,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
@@ -51,7 +51,6 @@ import org.elasticsearch.transport.BaseTransportResponseHandler;
 import org.elasticsearch.transport.NodeShouldNotConnectException;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
-import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestHandler;
 import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportService;
@@ -241,7 +240,7 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
                 throw globalBlockException;
             }
 
-            String[] concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, request);
+            String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request);
             ClusterBlockException requestBlockException = checkRequestBlock(clusterState, request, concreteIndices);
             if (requestBlockException != null) {
                 throw requestBlockException;
diff --git a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
index 4b19c3e6eba..1a7092c3a97 100644
--- a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
@@ -26,7 +26,6 @@ import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.ThreadedActionListener;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.MasterNodeChangePredicate;
@@ -34,6 +33,7 @@ import org.elasticsearch.cluster.NotMasterException;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.Discovery;
diff --git a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java
index ed08e5bdba3..5a6ddcfb34e 100644
--- a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java
@@ -21,9 +21,10 @@ package org.elasticsearch.action.support.master;
 
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -37,7 +38,8 @@ import java.util.function.Supplier;
 public abstract class TransportMasterNodeReadAction<Request extends MasterNodeReadRequest<Request>, Response extends ActionResponse>
         extends TransportMasterNodeAction<Request, Response> {
 
-    public static final Setting<Boolean> FORCE_LOCAL_SETTING = Setting.boolSetting("action.master.force_local", false, false, Setting.Scope.CLUSTER);
+    public static final Setting<Boolean> FORCE_LOCAL_SETTING =
+        Setting.boolSetting("action.master.force_local", false, Property.NodeScope);
 
     private final boolean forceLocal;
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java b/core/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java
index 7e42036c1d1..66b9fce5d71 100644
--- a/core/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java
@@ -22,9 +22,9 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -50,7 +50,7 @@ public abstract class TransportClusterInfoAction<Request extends ClusterInfoRequ
 
     @Override
     protected final void masterOperation(final Request request, final ClusterState state, final ActionListener<Response> listener) {
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
+        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request);
         doMasterOperation(request, concreteIndices, state, listener);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
index 7e2702afd8a..9c021efbe40 100644
--- a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
@@ -26,11 +26,11 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.ChildTaskRequest;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
index 6283e69a02e..b4cfbb6ad88 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
@@ -51,6 +51,8 @@ public abstract class ReplicationRequest<Request extends ReplicationRequest<Requ
      */
     protected ShardId shardId;
 
+    long primaryTerm;
+
     protected TimeValue timeout = DEFAULT_TIMEOUT;
     protected String index;
 
@@ -148,6 +150,16 @@ public abstract class ReplicationRequest<Request extends ReplicationRequest<Requ
         return routedBasedOnClusterVersion;
     }
 
+    /** returns the primary term active at the time the operation was performed on the primary shard */
+    public long primaryTerm() {
+        return primaryTerm;
+    }
+
+    /** marks the primary term in which the operation was performed */
+    public void primaryTerm(long term) {
+        primaryTerm = term;
+    }
+
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
@@ -169,6 +181,7 @@ public abstract class ReplicationRequest<Request extends ReplicationRequest<Requ
         timeout = TimeValue.readTimeValue(in);
         index = in.readString();
         routedBasedOnClusterVersion = in.readVLong();
+        primaryTerm = in.readVLong();
     }
 
     @Override
@@ -184,6 +197,7 @@ public abstract class ReplicationRequest<Request extends ReplicationRequest<Requ
         timeout.writeTo(out);
         out.writeString(index);
         out.writeVLong(routedBasedOnClusterVersion);
+        out.writeVLong(primaryTerm);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationTask.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationTask.java
index da3fce74fa8..9fe3da59a1f 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationTask.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationTask.java
@@ -59,7 +59,7 @@ public class ReplicationTask extends Task {
     }
 
     public static class Status implements Task.Status {
-        public static final Status PROTOTYPE = new Status("prototype");
+        public static final String NAME = "replication";
 
         private final String phase;
 
@@ -73,7 +73,7 @@ public class ReplicationTask extends Task {
 
         @Override
         public String getWriteableName() {
-            return "replication";
+            return NAME;
         }
 
         @Override
@@ -88,10 +88,5 @@ public class ReplicationTask extends Task {
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(phase);
         }
-
-        @Override
-        public Status readFrom(StreamInput in) throws IOException {
-            return new Status(in);
-        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
index a15819e82c7..25de821e227 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.action.support.replication;
 
 import com.carrotsearch.hppc.cursors.IntObjectCursor;
-
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ReplicationResponse;
@@ -32,11 +31,11 @@ import org.elasticsearch.action.support.TransportActions;
 import org.elasticsearch.action.support.broadcast.BroadcastRequest;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.index.shard.ShardId;
@@ -97,7 +96,7 @@ public abstract class TransportBroadcastReplicationAction<Request extends Broadc
                 @Override
                 public void onFailure(Throwable e) {
                     logger.trace("{}: got failure from {}", actionName, shardId);
-                    int totalNumCopies = clusterState.getMetaData().index(shardId.getIndexName()).getNumberOfReplicas() + 1;
+                    int totalNumCopies = clusterState.getMetaData().getIndexSafe(shardId.getIndex()).getNumberOfReplicas() + 1;
                     ShardResponse shardResponse = newShardResponse();
                     ReplicationResponse.ShardInfo.Failure[] failures;
                     if (TransportActions.isShardNotAvailableException(e)) {
@@ -130,7 +129,7 @@ public abstract class TransportBroadcastReplicationAction<Request extends Broadc
      */
     protected List<ShardId> shards(Request request, ClusterState clusterState) {
         List<ShardId> shardIds = new ArrayList<>();
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, request);
+        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request);
         for (String index : concreteIndices) {
             IndexMetaData indexMetaData = clusterState.metaData().getIndices().get(index);
             if (indexMetaData != null) {
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
index 96b70a08b83..4462caac0d0 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
@@ -28,10 +28,8 @@ import org.elasticsearch.action.WriteConsistencyLevel;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.action.support.TransportActions;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
-import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
@@ -43,6 +41,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -103,7 +102,6 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
     protected final ShardStateAction shardStateAction;
     protected final WriteConsistencyLevel defaultWriteConsistencyLevel;
     protected final TransportRequestOptions transportOptions;
-    protected final MappingUpdatedAction mappingUpdatedAction;
 
     final String transportReplicaAction;
     final String transportPrimaryAction;
@@ -113,7 +111,7 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
     protected TransportReplicationAction(Settings settings, String actionName, TransportService transportService,
                                          ClusterService clusterService, IndicesService indicesService,
                                          ThreadPool threadPool, ShardStateAction shardStateAction,
-                                         MappingUpdatedAction mappingUpdatedAction, ActionFilters actionFilters,
+                                         ActionFilters actionFilters,
                                          IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request,
                                          Supplier<ReplicaRequest> replicaRequest, String executor) {
         super(settings, actionName, threadPool, actionFilters, indexNameExpressionResolver, transportService.getTaskManager());
@@ -121,7 +119,6 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
         this.clusterService = clusterService;
         this.indicesService = indicesService;
         this.shardStateAction = shardStateAction;
-        this.mappingUpdatedAction = mappingUpdatedAction;
 
         this.transportPrimaryAction = actionName + "[p]";
         this.transportReplicaAction = actionName + "[r]";
@@ -270,7 +267,7 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
                     try {
                         channel.sendResponse(e);
                     } catch (Throwable e1) {
-                        logger.warn("Failed to send response for " + actionName, e1);
+                        logger.warn("Failed to send response for {}", e1, actionName);
                     }
                 }
             });
@@ -362,32 +359,7 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
                     }
                 });
             } else {
-                try {
-                    failReplicaIfNeeded(t);
-                } catch (Throwable unexpected) {
-                    logger.error("{} unexpected error while failing replica", unexpected, request.shardId().id());
-                } finally {
                     responseWithFailure(t);
-                }
-            }
-        }
-
-        private void failReplicaIfNeeded(Throwable t) {
-            String index = request.shardId().getIndex().getName();
-            int shardId = request.shardId().id();
-            logger.trace("failure on replica [{}][{}], action [{}], request [{}]", t, index, shardId, actionName, request);
-            if (ignoreReplicaException(t) == false) {
-                IndexService indexService = indicesService.indexService(index);
-                if (indexService == null) {
-                    logger.debug("ignoring failed replica [{}][{}] because index was already removed.", index, shardId);
-                    return;
-                }
-                IndexShard indexShard = indexService.getShardOrNull(shardId);
-                if (indexShard == null) {
-                    logger.debug("ignoring failed replica [{}][{}] because index was already removed.", index, shardId);
-                    return;
-                }
-                indexShard.failShard(actionName + " failed on replica", t);
             }
         }
 
@@ -395,7 +367,7 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
             try {
                 channel.sendResponse(t);
             } catch (IOException responseException) {
-                logger.warn("failed to send error message back to client for action [" + transportReplicaAction + "]", responseException);
+                logger.warn("failed to send error message back to client for action [{}]", responseException, transportReplicaAction);
                 logger.warn("actual Exception", t);
             }
         }
@@ -404,7 +376,7 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
         protected void doRun() throws Exception {
             setPhase(task, "replica");
             assert request.shardId() != null : "request shardId must be set";
-            try (Releasable ignored = getIndexShardReferenceOnReplica(request.shardId(), request)) {
+            try (Releasable ignored = getIndexShardReferenceOnReplica(request.shardId(), request.primaryTerm())) {
                 shardOperationOnReplica(request);
                 if (logger.isTraceEnabled()) {
                     logger.trace("action [{}] completed on shard [{}] for request [{}]", transportReplicaAction, request.shardId(), request);
@@ -525,7 +497,7 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
         }
 
         private String concreteIndex(ClusterState state) {
-            return resolveIndex() ? indexNameExpressionResolver.concreteSingleIndex(state, request) : request.index();
+            return resolveIndex() ? indexNameExpressionResolver.concreteSingleIndex(state, request).getName() : request.index();
         }
 
         private ShardRouting primary(ClusterState state) {
@@ -710,7 +682,6 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
             indexShardReference = getIndexShardReferenceOnPrimary(shardId, request);
             if (indexShardReference.isRelocated() == false) {
                 executeLocally();
-
             } else {
                 executeRemotely();
             }
@@ -719,6 +690,7 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
         private void executeLocally() throws Exception {
             // execute locally
             Tuple<Response, ReplicaRequest> primaryResponse = shardOperationOnPrimary(state.metaData(), request);
+            primaryResponse.v2().primaryTerm(indexShardReference.opPrimaryTerm());
             if (logger.isTraceEnabled()) {
                 logger.trace("action [{}] completed on shard [{}] for request [{}] with cluster state version [{}]", transportPrimaryAction, shardId, request, state.version());
             }
@@ -864,7 +836,7 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
     protected IndexShardReference getIndexShardReferenceOnPrimary(ShardId shardId, Request request) {
         IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
         IndexShard indexShard = indexService.getShard(shardId.id());
-        IndexShardReference ref = new IndexShardReferenceImpl(indexShard, true);
+        IndexShardReference ref = IndexShardReferenceImpl.createOnPrimary(indexShard);
         assert addShardReference(ref, "primary: " + request.toString() + " " + ref.routingEntry());
         return ref;
     }
@@ -873,11 +845,11 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
      * returns a new reference to {@link IndexShard} on a node that the request is replicated to. The reference is closed as soon as
      * replication is completed on the node.
      */
-    protected IndexShardReference getIndexShardReferenceOnReplica(ShardId shardId, ReplicaRequest request) {
+    protected IndexShardReference getIndexShardReferenceOnReplica(ShardId shardId, long primaryTerm) {
         IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
         IndexShard indexShard = indexService.getShard(shardId.id());
-        IndexShardReference ref = new IndexShardReferenceImpl(indexShard, false);
-        assert addShardReference(ref, "replica: " + request.toString() + " " + ref.routingEntry());
+        IndexShardReference ref = IndexShardReferenceImpl.createOnReplica(indexShard, primaryTerm);
+        assert addShardReference(ref, "primary term: " + primaryTerm + " " + ref.routingEntry());
         return ref;
     }
 
@@ -1147,13 +1119,17 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
                                 totalShards,
                                 success.get(),
                                 failuresArray
-
                         )
                 );
+                if (logger.isTraceEnabled()) {
+                    logger.trace("finished replicating action [{}], request [{}], shardInfo [{}]", actionName, replicaRequest,
+                            finalResponse.getShardInfo());
+                }
+
                 try {
                     channel.sendResponse(finalResponse);
                 } catch (IOException responseException) {
-                    logger.warn("failed to send error message back to client for action [" + transportReplicaAction + "]", responseException);
+                    logger.warn("failed to send error message back to client for action [{}]", responseException, transportReplicaAction);
                 }
                 if (logger.isTraceEnabled()) {
                     logger.trace("action [{}] completed on all replicas [{}] for request [{}]", transportReplicaAction, shardId, replicaRequest);
@@ -1176,6 +1152,9 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
         void failShard(String reason, @Nullable Throwable e);
 
         ShardRouting routingEntry();
+
+        /** returns the primary term of the current operation */
+        long opPrimaryTerm();
     }
 
     static final class IndexShardReferenceImpl implements IndexShardReference {
@@ -1183,15 +1162,23 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
         private final IndexShard indexShard;
         private final Releasable operationLock;
 
-        IndexShardReferenceImpl(IndexShard indexShard, boolean primaryAction) {
+        private IndexShardReferenceImpl(IndexShard indexShard, long primaryTerm) {
             this.indexShard = indexShard;
-            if (primaryAction) {
+            if (primaryTerm < 0) {
                 operationLock = indexShard.acquirePrimaryOperationLock();
             } else {
-                operationLock = indexShard.acquireReplicaOperationLock();
+                operationLock = indexShard.acquireReplicaOperationLock(primaryTerm);
             }
         }
 
+        static IndexShardReferenceImpl createOnPrimary(IndexShard indexShard) {
+            return new IndexShardReferenceImpl(indexShard, -1);
+        }
+
+        static IndexShardReferenceImpl createOnReplica(IndexShard indexShard, long primaryTerm) {
+            return new IndexShardReferenceImpl(indexShard, primaryTerm);
+        }
+
         @Override
         public void close() {
             operationLock.close();
@@ -1212,6 +1199,11 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
         public ShardRouting routingEntry() {
             return indexShard.routingEntry();
         }
+
+        @Override
+        public long opPrimaryTerm() {
+            return indexShard.getPrimaryTerm();
+        }
     }
 
     protected final void processAfterWrite(boolean refresh, IndexShard indexShard, Translog.Location location) {
diff --git a/core/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java b/core/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java
index 8f9e8fe0e25..186eb536b08 100644
--- a/core/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java
@@ -27,6 +27,7 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.shard.ShardId;
 
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
@@ -42,8 +43,8 @@ public abstract class InstanceShardOperationRequest<Request extends InstanceShar
     protected TimeValue timeout = DEFAULT_TIMEOUT;
 
     protected String index;
-    // -1 means its not set, allows to explicitly direct a request to a specific shard
-    protected int shardId = -1;
+    // null means its not set, allows to explicitly direct a request to a specific shard
+    protected ShardId shardId = null;
 
     private String concreteIndex;
 
@@ -115,7 +116,11 @@ public abstract class InstanceShardOperationRequest<Request extends InstanceShar
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         index = in.readString();
-        shardId = in.readInt();
+        if (in.readBoolean()) {
+            shardId = ShardId.readShardId(in);
+        } else {
+            shardId = null;
+        }
         timeout = TimeValue.readTimeValue(in);
         concreteIndex = in.readOptionalString();
     }
@@ -124,7 +129,7 @@ public abstract class InstanceShardOperationRequest<Request extends InstanceShar
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeString(index);
-        out.writeInt(shardId);
+        out.writeOptionalStreamable(shardId);
         timeout.writeTo(out);
         out.writeOptionalString(concreteIndex);
     }
diff --git a/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
index 545016828d1..fba7d80299b 100644
--- a/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.UnavailableShardsException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -34,6 +33,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -138,7 +138,7 @@ public abstract class TransportInstanceSingleOperationAction<Request extends Ins
                         throw blockException;
                     }
                 }
-                request.concreteIndex(indexNameExpressionResolver.concreteSingleIndex(observer.observedState(), request));
+                request.concreteIndex(indexNameExpressionResolver.concreteSingleIndex(observer.observedState(), request).getName());
                 resolveRequest(observer.observedState(), request);
                 blockException = checkRequestBlock(observer.observedState(), request);
                 if (blockException != null) {
@@ -172,7 +172,7 @@ public abstract class TransportInstanceSingleOperationAction<Request extends Ins
                 return;
             }
 
-            request.shardId = shardIt.shardId().id();
+            request.shardId = shardIt.shardId();
             DiscoveryNode node = nodes.get(shard.currentNodeId());
             transportService.sendRequest(node, shardActionName, request, transportOptions(), new BaseTransportResponseHandler<Response>() {
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java b/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
index 2d13b7f99ac..0c7f0627c66 100644
--- a/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
@@ -25,7 +25,6 @@ import org.elasticsearch.action.NoShardAvailableActionException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.action.support.TransportActions;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
@@ -34,6 +33,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.settings.Settings;
@@ -141,7 +141,7 @@ public abstract class TransportSingleShardAction<Request extends SingleShardRequ
 
             String concreteSingleIndex;
             if (resolveIndex(request)) {
-                concreteSingleIndex = indexNameExpressionResolver.concreteSingleIndex(clusterState, request);
+                concreteSingleIndex = indexNameExpressionResolver.concreteSingleIndex(clusterState, request).getName();
             } else {
                 concreteSingleIndex = request.index();
             }
diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java b/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java
index f7da48a667b..dc296a84720 100644
--- a/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java
@@ -60,7 +60,7 @@ public class BaseTasksRequest<Request extends BaseTasksRequest<Request>> extends
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
-        if (taskId.isSet() == false && nodesIds.length > 0) {
+        if (taskId.isSet() && nodesIds.length > 0) {
             validationException = addValidationError("task id cannot be used together with node ids",
                 validationException);
         }
@@ -71,7 +71,7 @@ public class BaseTasksRequest<Request extends BaseTasksRequest<Request>> extends
      * Sets the list of action masks for the actions that should be returned
      */
     @SuppressWarnings("unchecked")
-    public final Request actions(String... actions) {
+    public final Request setActions(String... actions) {
         this.actions = actions;
         return (Request) this;
     }
@@ -79,16 +79,16 @@ public class BaseTasksRequest<Request extends BaseTasksRequest<Request>> extends
     /**
      * Return the list of action masks for the actions that should be returned
      */
-    public String[] actions() {
+    public String[] getActions() {
         return actions;
     }
 
-    public final String[] nodesIds() {
+    public final String[] getNodesIds() {
         return nodesIds;
     }
 
     @SuppressWarnings("unchecked")
-    public final Request nodesIds(String... nodesIds) {
+    public final Request setNodesIds(String... nodesIds) {
         this.nodesIds = nodesIds;
         return (Request) this;
     }
@@ -98,12 +98,12 @@ public class BaseTasksRequest<Request extends BaseTasksRequest<Request>> extends
      *
      * By default tasks with any ids are returned.
      */
-    public TaskId taskId() {
+    public TaskId getTaskId() {
         return taskId;
     }
 
     @SuppressWarnings("unchecked")
-    public final Request taskId(TaskId taskId) {
+    public final Request setTaskId(TaskId taskId) {
         this.taskId = taskId;
         return (Request) this;
     }
@@ -112,29 +112,29 @@ public class BaseTasksRequest<Request extends BaseTasksRequest<Request>> extends
     /**
      * Returns the parent task id that tasks should be filtered by
      */
-    public TaskId parentTaskId() {
+    public TaskId getParentTaskId() {
         return parentTaskId;
     }
 
     @SuppressWarnings("unchecked")
-    public Request parentTaskId(TaskId parentTaskId) {
+    public Request setParentTaskId(TaskId parentTaskId) {
         this.parentTaskId = parentTaskId;
         return (Request) this;
     }
 
 
-    public TimeValue timeout() {
+    public TimeValue getTimeout() {
         return this.timeout;
     }
 
     @SuppressWarnings("unchecked")
-    public final Request timeout(TimeValue timeout) {
+    public final Request setTimeout(TimeValue timeout) {
         this.timeout = timeout;
         return (Request) this;
     }
 
     @SuppressWarnings("unchecked")
-    public final Request timeout(String timeout) {
+    public final Request setTimeout(String timeout) {
         this.timeout = TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout");
         return (Request) this;
     }
@@ -162,15 +162,15 @@ public class BaseTasksRequest<Request extends BaseTasksRequest<Request>> extends
     }
 
     public boolean match(Task task) {
-        if (actions() != null && actions().length > 0 && Regex.simpleMatch(actions(), task.getAction()) == false) {
+        if (getActions() != null && getActions().length > 0 && Regex.simpleMatch(getActions(), task.getAction()) == false) {
             return false;
         }
-        if (taskId().isSet() == false) {
-            if(taskId().getId() != task.getId()) {
+        if (getTaskId().isSet()) {
+            if(getTaskId().getId() != task.getId()) {
                 return false;
             }
         }
-        if (parentTaskId.isSet() == false) {
+        if (parentTaskId.isSet()) {
             if (parentTaskId.equals(task.getParentTaskId()) == false) {
                 return false;
             }
diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java
index a7265ce9998..a510a847c62 100644
--- a/core/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java
@@ -35,19 +35,19 @@ public class TasksRequestBuilder <Request extends BaseTasksRequest<Request>, Res
 
     @SuppressWarnings("unchecked")
     public final RequestBuilder setNodesIds(String... nodesIds) {
-        request.nodesIds(nodesIds);
+        request.setNodesIds(nodesIds);
         return (RequestBuilder) this;
     }
 
     @SuppressWarnings("unchecked")
     public final RequestBuilder setActions(String... actions) {
-        request.actions(actions);
+        request.setActions(actions);
         return (RequestBuilder) this;
     }
 
     @SuppressWarnings("unchecked")
     public final RequestBuilder setTimeout(TimeValue timeout) {
-        request.timeout(timeout);
+        request.setTimeout(timeout);
         return (RequestBuilder) this;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
index f10b9f23327..ad7702466cd 100644
--- a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
@@ -28,11 +28,11 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.ChildTaskRequest;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -124,25 +124,25 @@ public abstract class TransportTasksAction<
     }
 
     protected String[] resolveNodes(TasksRequest request, ClusterState clusterState) {
-        if (request.taskId().isSet()) {
-            return clusterState.nodes().resolveNodesIds(request.nodesIds());
+        if (request.getTaskId().isSet()) {
+            return new String[]{request.getTaskId().getNodeId()};
         } else {
-            return new String[]{request.taskId().getNodeId()};
+            return clusterState.nodes().resolveNodesIds(request.getNodesIds());
         }
     }
 
     protected void processTasks(TasksRequest request, Consumer<OperationTask> operation) {
-        if (request.taskId().isSet() == false) {
+        if (request.getTaskId().isSet()) {
             // we are only checking one task, we can optimize it
-            Task task = taskManager.getTask(request.taskId().getId());
+            Task task = taskManager.getTask(request.getTaskId().getId());
             if (task != null) {
                 if (request.match(task)) {
                     operation.accept((OperationTask) task);
                 } else {
-                    throw new ResourceNotFoundException("task [{}] doesn't support this operation", request.taskId());
+                    throw new ResourceNotFoundException("task [{}] doesn't support this operation", request.getTaskId());
                 }
             } else {
-                throw new ResourceNotFoundException("task [{}] is missing", request.taskId());
+                throw new ResourceNotFoundException("task [{}] is missing", request.getTaskId());
             }
         } else {
             for (Task task : taskManager.getTasks().values()) {
@@ -207,8 +207,8 @@ public abstract class TransportTasksAction<
             this.nodesIds = filterNodeIds(clusterState.nodes(), nodesIds);
             ImmutableOpenMap<String, DiscoveryNode> nodes = clusterState.nodes().nodes();
             this.nodes = new DiscoveryNode[nodesIds.length];
-            for (int i = 0; i < nodesIds.length; i++) {
-                this.nodes[i] = nodes.get(nodesIds[i]);
+            for (int i = 0; i < this.nodesIds.length; i++) {
+                this.nodes[i] = nodes.get(this.nodesIds[i]);
             }
             this.responses = new AtomicReferenceArray<>(this.nodesIds.length);
         }
@@ -224,8 +224,8 @@ public abstract class TransportTasksAction<
                 }
             } else {
                 TransportRequestOptions.Builder builder = TransportRequestOptions.builder();
-                if (request.timeout() != null) {
-                    builder.withTimeout(request.timeout());
+                if (request.getTimeout() != null) {
+                    builder.withTimeout(request.getTimeout());
                 }
                 builder.withCompress(transportCompress());
                 for (int i = 0; i < nodesIds.length; i++) {
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java b/core/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java
index 7047ee69040..d71958cefde 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java
@@ -20,13 +20,12 @@
 package org.elasticsearch.action.termvectors;
 
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.DocumentRequest;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
@@ -72,7 +71,7 @@ public class TransportMultiTermVectorsAction extends HandledTransportAction<Mult
                         termVectorsRequest.type(), termVectorsRequest.id(), new IndexNotFoundException(termVectorsRequest.index()))));
                 continue;
             }
-            String concreteSingleIndex = indexNameExpressionResolver.concreteSingleIndex(clusterState, (DocumentRequest) termVectorsRequest);
+            String concreteSingleIndex = indexNameExpressionResolver.concreteSingleIndex(clusterState, termVectorsRequest).getName();
             if (termVectorsRequest.routing() == null && clusterState.getMetaData().routingRequired(concreteSingleIndex, termVectorsRequest.type())) {
                 responses.set(i, new MultiTermVectorsItemResponse(null, new MultiTermVectorsResponse.Failure(concreteSingleIndex, termVectorsRequest.type(), termVectorsRequest.id(),
                         new IllegalArgumentException("routing is required for [" + concreteSingleIndex + "]/[" + termVectorsRequest.type() + "]/[" + termVectorsRequest.id() + "]"))));
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java b/core/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java
index ccdf934958d..197ab4260b2 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java
@@ -23,10 +23,10 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportActions;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardIterator;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexService;
@@ -75,12 +75,12 @@ public class TransportShardMultiTermsVectorAction extends TransportSingleShardAc
 
     @Override
     protected MultiTermVectorsShardResponse shardOperation(MultiTermVectorsShardRequest request, ShardId shardId) {
-        MultiTermVectorsShardResponse response = new MultiTermVectorsShardResponse();
+        final MultiTermVectorsShardResponse response = new MultiTermVectorsShardResponse();
+        final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
+        final IndexShard indexShard = indexService.getShard(shardId.id());
         for (int i = 0; i < request.locations.size(); i++) {
             TermVectorsRequest termVectorsRequest = request.requests.get(i);
             try {
-                IndexService indexService = indicesService.indexServiceSafe(request.index());
-                IndexShard indexShard = indexService.getShard(shardId.id());
                 TermVectorsResponse termVectorsResponse = TermVectorsService.getTermVectors(indexShard, termVectorsRequest);
                 termVectorsResponse.updateTookInMillis(termVectorsRequest.startTime());
                 response.add(request.locations.get(i), termVectorsResponse);
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java b/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
index 5b0b9fd2726..da8754542ab 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
@@ -23,10 +23,10 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardIterator;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexService;
diff --git a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
index 0aefa825f2a..0363ef8fe43 100644
--- a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
@@ -35,13 +35,13 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.action.support.TransportActions;
 import org.elasticsearch.action.support.single.instance.TransportInstanceSingleOperationAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.routing.PlainShardIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
@@ -51,6 +51,7 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -147,8 +148,8 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
 
     @Override
     protected ShardIterator shards(ClusterState clusterState, UpdateRequest request) {
-        if (request.shardId() != -1) {
-            return clusterState.routingTable().index(request.concreteIndex()).shard(request.shardId()).primaryShardIt();
+        if (request.getShardId() != null) {
+            return clusterState.routingTable().index(request.concreteIndex()).shard(request.getShardId().getId()).primaryShardIt();
         }
         ShardIterator shardIterator = clusterService.operationRouting()
                 .indexShards(clusterState, request.concreteIndex(), request.type(), request.id(), request.routing());
@@ -167,8 +168,9 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
     }
 
     protected void shardOperation(final UpdateRequest request, final ActionListener<UpdateResponse> listener, final int retryCount) {
-        final IndexService indexService = indicesService.indexServiceSafe(request.concreteIndex());
-        final IndexShard indexShard = indexService.getShard(request.shardId());
+        final ShardId shardId = request.getShardId();
+        final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
+        final IndexShard indexShard = indexService.getShard(shardId.getId());
         final UpdateHelper.Result result = updateHelper.prepare(request, indexShard);
         switch (result.operation()) {
             case UPSERT:
@@ -194,7 +196,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
                         if (e instanceof VersionConflictEngineException) {
                             if (retryCount < request.retryOnConflict()) {
                                 logger.trace("Retry attempt [{}] of [{}] on version conflict on [{}][{}][{}]",
-                                        retryCount + 1, request.retryOnConflict(), request.index(), request.shardId(), request.id());
+                                        retryCount + 1, request.retryOnConflict(), request.index(), request.getShardId(), request.id());
                                 threadPool.executor(executor()).execute(new ActionRunnable<UpdateResponse>(listener) {
                                     @Override
                                     protected void doRun() {
@@ -267,9 +269,9 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
                 break;
             case NONE:
                 UpdateResponse update = result.action();
-                IndexService indexServiceOrNull = indicesService.indexService(request.concreteIndex());
+                IndexService indexServiceOrNull = indicesService.indexService(shardId.getIndex());
                 if (indexServiceOrNull !=  null) {
-                    IndexShard shard = indexService.getShardOrNull(request.shardId());
+                    IndexShard shard = indexService.getShardOrNull(shardId.getId());
                     if (shard != null) {
                         shard.noopUpdate(request.type());
                     }
diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
index 6bc69ed4d9c..0877ea1c66b 100644
--- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
@@ -36,6 +36,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptParameterParser;
 import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue;
@@ -43,6 +44,7 @@ import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptService.ScriptType;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -88,7 +90,7 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
     }
 
     public UpdateRequest(String index, String type, String id) {
-        this.index = index;
+        super(index);
         this.type = type;
         this.id = id;
     }
@@ -195,7 +197,7 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
         return parent;
     }
 
-    int shardId() {
+    public ShardId getShardId() {
         return this.shardId;
     }
 
@@ -670,9 +672,15 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
                 } else if ("detect_noop".equals(currentFieldName)) {
                     detectNoop(parser.booleanValue());
                 } else if ("fields".equals(currentFieldName)) {
-                    List<Object> values = parser.list();
-                    String[] fields = values.toArray(new String[values.size()]);
-                    fields(fields);
+                    List<Object> fields = null;
+                    if (token == XContentParser.Token.START_ARRAY) {
+                        fields = (List) parser.list();
+                    } else if (token.isValue()) {
+                        fields = Collections.singletonList(parser.text());
+                    }
+                    if (fields != null) {
+                        fields(fields.toArray(new String[fields.size()]));
+                    }
                 } else {
                     //here we don't have settings available, unable to throw deprecation exceptions
                     scriptParameterParser.token(currentFieldName, token, parser, ParseFieldMatcher.EMPTY);
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
index 215659054d2..2cb4fb6450a 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -24,10 +24,10 @@ import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.StringHelper;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
+import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.common.PidFile;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.cli.CliTool;
-import org.elasticsearch.common.cli.Terminal;
 import org.elasticsearch.common.inject.CreationException;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.LogConfigurator;
@@ -45,10 +45,9 @@ import java.io.IOException;
 import java.io.PrintStream;
 import java.nio.file.Path;
 import java.util.Locale;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 
-import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
-
 /**
  * Internal startup code.
  */
@@ -136,6 +135,8 @@ final class Bootstrap {
 
         JNANatives.trySetMaxNumberOfThreads();
 
+        JNANatives.trySetMaxSizeVirtualMemory();
+
         // init lucene random seed. it will use /dev/urandom where available:
         StringHelper.randomId();
     }
@@ -189,9 +190,13 @@ final class Bootstrap {
         node = new Node(nodeSettings);
     }
 
-    private static Environment initialSettings(boolean foreground) {
+    private static Environment initialSettings(boolean foreground, String pidFile) {
         Terminal terminal = foreground ? Terminal.DEFAULT : null;
-        return InternalSettingsPreparer.prepareEnvironment(EMPTY_SETTINGS, terminal);
+        Settings.Builder builder = Settings.builder();
+        if (Strings.hasLength(pidFile)) {
+            builder.put(Environment.PIDFILE_SETTING.getKey(), pidFile);
+        }
+        return InternalSettingsPreparer.prepareEnvironment(builder.build(), terminal);
     }
 
     private void start() {
@@ -218,22 +223,18 @@ final class Bootstrap {
      * This method is invoked by {@link Elasticsearch#main(String[])}
      * to startup elasticsearch.
      */
-    static void init(String[] args) throws Throwable {
+    static void init(
+            final boolean foreground,
+            final String pidFile,
+            final Map<String, String> esSettings) throws Throwable {
         // Set the system property before anything has a chance to trigger its use
         initLoggerPrefix();
 
-        BootstrapCLIParser bootstrapCLIParser = new BootstrapCLIParser();
-        CliTool.ExitStatus status = bootstrapCLIParser.execute(args);
-
-        if (CliTool.ExitStatus.OK != status) {
-            exit(status.status());
-        }
+        elasticsearchSettings(esSettings);
 
         INSTANCE = new Bootstrap();
 
-        boolean foreground = !"false".equals(System.getProperty("es.foreground", System.getProperty("es-foreground")));
-
-        Environment environment = initialSettings(foreground);
+        Environment environment = initialSettings(foreground, pidFile);
         Settings settings = environment.settings();
         LogConfigurator.configure(settings, true);
         checkForCustomConfFile();
@@ -297,6 +298,13 @@ final class Bootstrap {
         }
     }
 
+    @SuppressForbidden(reason = "Sets system properties passed as CLI parameters")
+    private static void elasticsearchSettings(Map<String, String> esSettings) {
+        for (Map.Entry<String, String> esSetting : esSettings.entrySet()) {
+            System.setProperty(esSetting.getKey(), esSetting.getValue());
+        }
+    }
+
     @SuppressForbidden(reason = "System#out")
     private static void closeSystOut() {
         System.out.close();
@@ -307,14 +315,6 @@ final class Bootstrap {
         System.err.close();
     }
 
-    @SuppressForbidden(reason = "System#err")
-    private static void sysError(String line, boolean flush) {
-        System.err.println(line);
-        if (flush) {
-            System.err.flush();
-        }
-    }
-
     private static void checkForCustomConfFile() {
         String confFileSetting = System.getProperty("es.default.config");
         checkUnsetAndMaybeExit(confFileSetting, "es.default.config");
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCLIParser.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCLIParser.java
deleted file mode 100644
index 25ae53873fe..00000000000
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCLIParser.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.bootstrap;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.elasticsearch.Build;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.cli.CliTool;
-import org.elasticsearch.common.cli.CliToolConfig;
-import org.elasticsearch.common.cli.UserError;
-import org.elasticsearch.common.cli.Terminal;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.monitor.jvm.JvmInfo;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Properties;
-
-import static org.elasticsearch.common.cli.CliToolConfig.Builder.cmd;
-import static org.elasticsearch.common.cli.CliToolConfig.Builder.optionBuilder;
-
-final class BootstrapCLIParser extends CliTool {
-
-    private static final CliToolConfig CONFIG = CliToolConfig.config("elasticsearch", BootstrapCLIParser.class)
-            .cmds(Start.CMD, Version.CMD)
-            .build();
-
-    public BootstrapCLIParser() {
-        super(CONFIG);
-    }
-
-    public BootstrapCLIParser(Terminal terminal) {
-        super(CONFIG, terminal);
-    }
-
-    @Override
-    protected Command parse(String cmdName, CommandLine cli) throws Exception {
-        switch (cmdName.toLowerCase(Locale.ROOT)) {
-            case Start.NAME:
-                return Start.parse(terminal, cli);
-            case Version.NAME:
-                return Version.parse(terminal, cli);
-            default:
-                assert false : "should never get here, if the user enters an unknown command, an error message should be shown before parse is called";
-                return null;
-        }
-    }
-
-    static class Version extends CliTool.Command {
-
-        private static final String NAME = "version";
-
-        private static final CliToolConfig.Cmd CMD = cmd(NAME, Version.class).build();
-
-        public static Command parse(Terminal terminal, CommandLine cli) {
-            return new Version(terminal);
-        }
-
-        public Version(Terminal terminal) {
-            super(terminal);
-        }
-
-        @Override
-        public ExitStatus execute(Settings settings, Environment env) throws Exception {
-            terminal.println("Version: " + org.elasticsearch.Version.CURRENT
-                    + ", Build: " + Build.CURRENT.shortHash() + "/" + Build.CURRENT.date()
-                    + ", JVM: " + JvmInfo.jvmInfo().version());
-            return ExitStatus.OK_AND_EXIT;
-        }
-    }
-
-    static class Start extends CliTool.Command {
-
-        private static final String NAME = "start";
-
-        private static final CliToolConfig.Cmd CMD = cmd(NAME, Start.class)
-                .options(
-                        optionBuilder("d", "daemonize").hasArg(false).required(false),
-                        optionBuilder("p", "pidfile").hasArg(true).required(false),
-                        optionBuilder("V", "version").hasArg(false).required(false),
-                        Option.builder("D").argName("property=value").valueSeparator('=').numberOfArgs(2)
-                )
-                .stopAtNonOption(true) // needed to parse the --foo.bar options, so this parser must be lenient
-                .build();
-
-        // TODO: don't use system properties as a way to do this, its horrible...
-        @SuppressForbidden(reason = "Sets system properties passed as CLI parameters")
-        public static Command parse(Terminal terminal, CommandLine cli) throws UserError {
-            if (cli.hasOption("V")) {
-                return Version.parse(terminal, cli);
-            }
-
-            if (cli.hasOption("d")) {
-                System.setProperty("es.foreground", "false");
-            }
-
-            String pidFile = cli.getOptionValue("pidfile");
-            if (!Strings.isNullOrEmpty(pidFile)) {
-                System.setProperty("es.pidfile", pidFile);
-            }
-
-            if (cli.hasOption("D")) {
-                Properties properties = cli.getOptionProperties("D");
-                for (Map.Entry<Object, Object> entry : properties.entrySet()) {
-                    String key = (String) entry.getKey();
-                    String propertyName = key.startsWith("es.") ? key : "es." + key;
-                    System.setProperty(propertyName, entry.getValue().toString());
-                }
-            }
-
-            // hacky way to extract all the fancy extra args, there is no CLI tool helper for this
-            Iterator<String> iterator = cli.getArgList().iterator();
-            final Map<String, String> properties = new HashMap<>();
-            while (iterator.hasNext()) {
-                String arg = iterator.next();
-                if (!arg.startsWith("--")) {
-                    if (arg.startsWith("-D") || arg.startsWith("-d") || arg.startsWith("-p")) {
-                        throw new UserError(ExitStatus.USAGE,
-                                "Parameter [" + arg + "] starting with \"-D\", \"-d\" or \"-p\" must be before any parameters starting with --"
-                        );
-                    } else {
-                        throw new UserError(ExitStatus.USAGE, "Parameter [" + arg + "]does not start with --");
-                    }
-                }
-                // if there is no = sign, we have to get the next argu
-                arg = arg.replace("--", "");
-                if (arg.contains("=")) {
-                    String[] splitArg = arg.split("=", 2);
-                    String key = splitArg[0];
-                    String value = splitArg[1];
-                    properties.put("es." + key, value);
-                } else {
-                    if (iterator.hasNext()) {
-                        String value = iterator.next();
-                        if (value.startsWith("--")) {
-                            throw new UserError(ExitStatus.USAGE, "Parameter [" + arg + "] needs value");
-                        }
-                        properties.put("es." + arg, value);
-                    } else {
-                        throw new UserError(ExitStatus.USAGE, "Parameter [" + arg + "] needs value");
-                    }
-                }
-            }
-            for (Map.Entry<String, String> entry : properties.entrySet()) {
-                System.setProperty(entry.getKey(), entry.getValue());
-            }
-            return new Start(terminal);
-        }
-
-        public Start(Terminal terminal) {
-            super(terminal);
-
-        }
-
-        @Override
-        public ExitStatus execute(Settings settings, Environment env) throws Exception {
-            return ExitStatus.OK;
-        }
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
index 433dd4498a4..dcb88dca848 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.monitor.process.ProcessProbe;
 import org.elasticsearch.transport.TransportSettings;
 
@@ -39,7 +40,6 @@ import java.util.Set;
 /**
  * We enforce limits once any network host is configured. In this case we assume the node is running in production
  * and all production limit checks must pass. This should be extended as we go to settings like:
- * - discovery.zen.minimum_master_nodes
  * - discovery.zen.ping.unicast.hosts is set if we use zen disco
  * - ensure we can write in all data directories
  * - fail if vm.max_map_count is under a certain limit (not sure if this works cross platform)
@@ -114,15 +114,19 @@ final class BootstrapCheck {
     }
 
     // the list of checks to execute
-    private static List<Check> checks(final Settings settings) {
+    static List<Check> checks(final Settings settings) {
         final List<Check> checks = new ArrayList<>();
         final FileDescriptorCheck fileDescriptorCheck
-                = Constants.MAC_OS_X ? new OsXFileDescriptorCheck() : new FileDescriptorCheck();
+            = Constants.MAC_OS_X ? new OsXFileDescriptorCheck() : new FileDescriptorCheck();
         checks.add(fileDescriptorCheck);
         checks.add(new MlockallCheck(BootstrapSettings.MLOCKALL_SETTING.get(settings)));
         if (Constants.LINUX) {
             checks.add(new MaxNumberOfThreadsCheck());
         }
+        if (Constants.LINUX || Constants.MAC_OS_X) {
+            checks.add(new MaxSizeVirtualMemoryCheck());
+        }
+        checks.add(new MinMasterNodesCheck(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(settings)));
         return Collections.unmodifiableList(checks);
     }
 
@@ -183,10 +187,10 @@ final class BootstrapCheck {
         @Override
         public final String errorMessage() {
             return String.format(
-                    Locale.ROOT,
-                    "max file descriptors [%d] for elasticsearch process likely too low, increase to at least [%d]",
-                    getMaxFileDescriptorCount(),
-                    limit
+                Locale.ROOT,
+                "max file descriptors [%d] for elasticsearch process likely too low, increase to at least [%d]",
+                getMaxFileDescriptorCount(),
+                limit
             );
         }
 
@@ -223,6 +227,26 @@ final class BootstrapCheck {
 
     }
 
+    static class MinMasterNodesCheck implements Check {
+
+        final boolean minMasterNodesIsSet;
+
+        MinMasterNodesCheck(boolean minMasterNodesIsSet) {
+            this.minMasterNodesIsSet = minMasterNodesIsSet;
+        }
+
+        @Override
+        public boolean check() {
+            return minMasterNodesIsSet == false;
+        }
+
+        @Override
+        public String errorMessage() {
+            return "please set [" + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() +
+                "] to a majority of the number of master eligible nodes in your cluster.";
+        }
+    }
+
     static class MaxNumberOfThreadsCheck implements Check {
 
         private final long maxNumberOfThreadsThreshold = 1 << 11;
@@ -249,4 +273,32 @@ final class BootstrapCheck {
 
     }
 
+    static class MaxSizeVirtualMemoryCheck implements Check {
+
+        @Override
+        public boolean check() {
+            return getMaxSizeVirtualMemory() != Long.MIN_VALUE && getMaxSizeVirtualMemory() != getRlimInfinity();
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                Locale.ROOT,
+                "max size virtual memory [%d] for user [%s] likely too low, increase to [unlimited]",
+                getMaxSizeVirtualMemory(),
+                BootstrapInfo.getSystemProperties().get("user.name"));
+        }
+
+        // visible for testing
+        long getRlimInfinity() {
+            return JNACLibrary.RLIM_INFINITY;
+        }
+
+        // visible for testing
+        long getMaxSizeVirtualMemory() {
+            return JNANatives.MAX_SIZE_VIRTUAL_MEMORY;
+        }
+
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java
index a20ff9bb059..4e9dffc995b 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.bootstrap;
 
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Scope;
+import org.elasticsearch.common.settings.Setting.Property;
 
 public final class BootstrapSettings {
 
@@ -29,10 +29,13 @@ public final class BootstrapSettings {
 
     // TODO: remove this hack when insecure defaults are removed from java
     public static final Setting<Boolean> SECURITY_FILTER_BAD_DEFAULTS_SETTING =
-            Setting.boolSetting("security.manager.filter_bad_defaults", true, false, Scope.CLUSTER);
+            Setting.boolSetting("security.manager.filter_bad_defaults", true, Property.NodeScope);
 
-    public static final Setting<Boolean> MLOCKALL_SETTING = Setting.boolSetting("bootstrap.mlockall", false, false, Scope.CLUSTER);
-    public static final Setting<Boolean> SECCOMP_SETTING = Setting.boolSetting("bootstrap.seccomp", true, false, Scope.CLUSTER);
-    public static final Setting<Boolean> CTRLHANDLER_SETTING = Setting.boolSetting("bootstrap.ctrlhandler", true, false, Scope.CLUSTER);
+    public static final Setting<Boolean> MLOCKALL_SETTING =
+        Setting.boolSetting("bootstrap.mlockall", false, Property.NodeScope);
+    public static final Setting<Boolean> SECCOMP_SETTING =
+        Setting.boolSetting("bootstrap.seccomp", true, Property.NodeScope);
+    public static final Setting<Boolean> CTRLHANDLER_SETTING =
+        Setting.boolSetting("bootstrap.ctrlhandler", true, Property.NodeScope);
 
 }
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
index 107a955696c..bb1f6cc87d5 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
@@ -19,23 +19,97 @@
 
 package org.elasticsearch.bootstrap;
 
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import joptsimple.util.KeyValuePair;
+import org.elasticsearch.Build;
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserError;
+import org.elasticsearch.monitor.jvm.JvmInfo;
+
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
 
 /**
  * This class starts elasticsearch.
  */
-public final class Elasticsearch {
+class Elasticsearch extends Command {
 
-    /** no instantiation */
-    private Elasticsearch() {}
+    private final OptionSpec<Void> versionOption;
+    private final OptionSpec<Void> daemonizeOption;
+    private final OptionSpec<String> pidfileOption;
+    private final OptionSpec<KeyValuePair> propertyOption;
+
+    // visible for testing
+    Elasticsearch() {
+        super("starts elasticsearch");
+        // TODO: in jopt-simple 5.0, make this mutually exclusive with all other options
+        versionOption = parser.acceptsAll(Arrays.asList("V", "version"),
+            "Prints elasticsearch version information and exits");
+        daemonizeOption = parser.acceptsAll(Arrays.asList("d", "daemonize"),
+            "Starts Elasticsearch in the background");
+        // TODO: in jopt-simple 5.0 this option type can be a Path
+        pidfileOption = parser.acceptsAll(Arrays.asList("p", "pidfile"),
+            "Creates a pid file in the specified path on start")
+            .withRequiredArg();
+        propertyOption = parser.accepts("E", "Configure an Elasticsearch setting").withRequiredArg().ofType(KeyValuePair.class);
+    }
 
     /**
      * Main entry point for starting elasticsearch
      */
-    public static void main(String[] args) throws StartupError {
+    public static void main(final String[] args) throws Exception {
+        final Elasticsearch elasticsearch = new Elasticsearch();
+        int status = main(args, elasticsearch, Terminal.DEFAULT);
+        if (status != ExitCodes.OK) {
+            exit(status);
+        }
+    }
+
+    static int main(final String[] args, final Elasticsearch elasticsearch, final Terminal terminal) throws Exception {
+        return elasticsearch.main(args, terminal);
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options) throws Exception {
+        if (options.nonOptionArguments().isEmpty() == false) {
+            throw new UserError(ExitCodes.USAGE, "Positional arguments not allowed, found " + options.nonOptionArguments());
+        }
+        if (options.has(versionOption)) {
+            if (options.has(daemonizeOption) || options.has(pidfileOption)) {
+                throw new UserError(ExitCodes.USAGE, "Elasticsearch version option is mutually exclusive with any other option");
+            }
+            terminal.println("Version: " + org.elasticsearch.Version.CURRENT
+                + ", Build: " + Build.CURRENT.shortHash() + "/" + Build.CURRENT.date()
+                + ", JVM: " + JvmInfo.jvmInfo().version());
+            return;
+        }
+
+        final boolean daemonize = options.has(daemonizeOption);
+        final String pidFile = pidfileOption.value(options);
+
+        final Map<String, String> esSettings = new HashMap<>();
+        for (final KeyValuePair kvp : propertyOption.values(options)) {
+            if (!kvp.key.startsWith("es.")) {
+                throw new UserError(ExitCodes.USAGE, "Elasticsearch settings must be prefixed with [es.] but was [" + kvp.key + "]");
+            }
+            if (kvp.value.isEmpty()) {
+                throw new UserError(ExitCodes.USAGE, "Elasticsearch setting [" + kvp.key + "] must not be empty");
+            }
+            esSettings.put(kvp.key, kvp.value);
+        }
+
+        init(daemonize, pidFile, esSettings);
+    }
+
+    void init(final boolean daemonize, final String pidFile, final Map<String, String> esSettings) {
         try {
-            Bootstrap.init(args);
-        } catch (Throwable t) {
+            Bootstrap.init(!daemonize, pidFile, esSettings);
+        } catch (final Throwable t) {
             // format exceptions to the console in a special way
             // to avoid 2MB stacktraces from guice, etc.
             throw new StartupError(t);
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java b/core/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java
index 573f3d5be3e..5d1369b21f7 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java
@@ -39,6 +39,7 @@ final class JNACLibrary {
     public static final int MCL_CURRENT = 1;
     public static final int ENOMEM = 12;
     public static final int RLIMIT_MEMLOCK = Constants.MAC_OS_X ? 6 : 8;
+    public static final int RLIMIT_AS = Constants.MAC_OS_X ? 5 : 9;
     public static final long RLIM_INFINITY = Constants.MAC_OS_X ? 9223372036854775807L : -1L;
 
     static {
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java b/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java
index b9d5ce11dbc..e55d38a0f72 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java
@@ -52,6 +52,8 @@ class JNANatives {
     // the user ID that owns the running Elasticsearch process
     static long MAX_NUMBER_OF_THREADS = -1;
 
+    static long MAX_SIZE_VIRTUAL_MEMORY = Long.MIN_VALUE;
+
     static void tryMlockall() {
         int errno = Integer.MIN_VALUE;
         String errMsg = null;
@@ -76,7 +78,7 @@ class JNANatives {
                     softLimit = rlimit.rlim_cur.longValue();
                     hardLimit = rlimit.rlim_max.longValue();
                 } else {
-                    logger.warn("Unable to retrieve resource limits: " + JNACLibrary.strerror(Native.getLastError()));
+                    logger.warn("Unable to retrieve resource limits: {}", JNACLibrary.strerror(Native.getLastError()));
                 }
             }
         } catch (UnsatisfiedLinkError e) {
@@ -85,19 +87,20 @@ class JNANatives {
         }
 
         // mlockall failed for some reason
-        logger.warn("Unable to lock JVM Memory: error=" + errno + ",reason=" + errMsg);
+        logger.warn("Unable to lock JVM Memory: error={}, reason={}", errno , errMsg);
         logger.warn("This can result in part of the JVM being swapped out.");
         if (errno == JNACLibrary.ENOMEM) {
             if (rlimitSuccess) {
-                logger.warn("Increase RLIMIT_MEMLOCK, soft limit: " + rlimitToString(softLimit) + ", hard limit: " + rlimitToString(hardLimit));
+                logger.warn("Increase RLIMIT_MEMLOCK, soft limit: {}, hard limit: {}", rlimitToString(softLimit), rlimitToString(hardLimit));
                 if (Constants.LINUX) {
                     // give specific instructions for the linux case to make it easy
                     String user = System.getProperty("user.name");
                     logger.warn("These can be adjusted by modifying /etc/security/limits.conf, for example: \n" +
-                                "\t# allow user '" + user + "' mlockall\n" +
-                                "\t" + user + " soft memlock unlimited\n" +
-                                "\t" + user + " hard memlock unlimited"
-                               );
+                                "\t# allow user '{}' mlockall\n" +
+                                "\t{} soft memlock unlimited\n" +
+                                "\t{} hard memlock unlimited",
+                                user, user, user
+                                );
                     logger.warn("If you are logged in interactively, you will have to re-login for the new limits to take effect.");
                 }
             } else {
@@ -123,6 +126,17 @@ class JNANatives {
         }
     }
 
+    static void trySetMaxSizeVirtualMemory() {
+        if (Constants.LINUX || Constants.MAC_OS_X) {
+            final JNACLibrary.Rlimit rlimit = new JNACLibrary.Rlimit();
+            if (JNACLibrary.getrlimit(JNACLibrary.RLIMIT_AS, rlimit) == 0) {
+                MAX_SIZE_VIRTUAL_MEMORY = rlimit.rlim_cur.longValue();
+            } else {
+                logger.warn("unable to retrieve max size virtual memory [" + JNACLibrary.strerror(Native.getLastError()) + "]");
+            }
+        }
+    }
+
     static String rlimitToString(long value) {
         assert Constants.LINUX || Constants.MAC_OS_X;
         if (value == JNACLibrary.RLIM_INFINITY) {
@@ -155,7 +169,7 @@ class JNANatives {
             // the amount of memory we wish to lock, plus a small overhead (1MB).
             SizeT size = new SizeT(JvmInfo.jvmInfo().getMem().getHeapInit().getBytes() + (1024 * 1024));
             if (!kernel.SetProcessWorkingSetSize(process, size, size)) {
-                logger.warn("Unable to lock JVM memory. Failed to set working set size. Error code " + Native.getLastError());
+                logger.warn("Unable to lock JVM memory. Failed to set working set size. Error code {}", Native.getLastError());
             } else {
                 JNAKernel32Library.MemoryBasicInformation memInfo = new JNAKernel32Library.MemoryBasicInformation();
                 long address = 0;
@@ -188,7 +202,7 @@ class JNANatives {
                 if (result) {
                     logger.debug("console ctrl handler correctly set");
                 } else {
-                    logger.warn("unknown error " + Native.getLastError() + " when adding console ctrl handler:");
+                    logger.warn("unknown error {} when adding console ctrl handler", Native.getLastError());
                 }
             } catch (UnsatisfiedLinkError e) {
                 // this will have already been logged by Kernel32Library, no need to repeat it
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JVMCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/JVMCheck.java
index 3f81cd035bd..86629e4fa36 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/JVMCheck.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/JVMCheck.java
@@ -200,7 +200,7 @@ final class JVMCheck {
             HotSpotCheck bug = JVM_BROKEN_HOTSPOT_VERSIONS.get(Constants.JVM_VERSION);
             if (bug != null && bug.check()) {
                 if (bug.getWorkaround().isPresent() && ManagementFactory.getRuntimeMXBean().getInputArguments().contains(bug.getWorkaround().get())) {
-                    Loggers.getLogger(JVMCheck.class).warn(bug.getWarningMessage().get());
+                    Loggers.getLogger(JVMCheck.class).warn("{}", bug.getWarningMessage().get());
                 } else {
                     throw new RuntimeException(bug.getErrorMessage());
                 }
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java b/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java
index 4325c5b7aef..46908e60642 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java
@@ -394,7 +394,7 @@ final class Seccomp {
             method = 0;
             int errno1 = Native.getLastError();
             if (logger.isDebugEnabled()) {
-                logger.debug("seccomp(SECCOMP_SET_MODE_FILTER): " + JNACLibrary.strerror(errno1) + ", falling back to prctl(PR_SET_SECCOMP)...");
+                logger.debug("seccomp(SECCOMP_SET_MODE_FILTER): {}, falling back to prctl(PR_SET_SECCOMP)...", JNACLibrary.strerror(errno1));
             }
             if (linux_prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, pointer, 0, 0) != 0) {
                 int errno2 = Native.getLastError();
diff --git a/core/src/main/java/org/elasticsearch/cache/recycler/PageCacheRecycler.java b/core/src/main/java/org/elasticsearch/cache/recycler/PageCacheRecycler.java
index a293428192b..2fad8678649 100644
--- a/core/src/main/java/org/elasticsearch/cache/recycler/PageCacheRecycler.java
+++ b/core/src/main/java/org/elasticsearch/cache/recycler/PageCacheRecycler.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.cache.recycler;
 
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.recycler.AbstractRecyclerC;
 import org.elasticsearch.common.recycler.Recycler;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
@@ -43,13 +43,19 @@ import static org.elasticsearch.common.recycler.Recyclers.none;
 /** A recycler of fixed-size pages. */
 public class PageCacheRecycler extends AbstractComponent implements Releasable {
 
-    public static final Setting<Type> TYPE_SETTING = new Setting<>("cache.recycler.page.type", Type.CONCURRENT.name(), Type::parse, false, Setting.Scope.CLUSTER);
-    public static final Setting<ByteSizeValue> LIMIT_HEAP_SETTING  = Setting.byteSizeSetting("cache.recycler.page.limit.heap", "10%", false, Setting.Scope.CLUSTER);
-    public static final Setting<Double> WEIGHT_BYTES_SETTING  = Setting.doubleSetting("cache.recycler.page.weight.bytes", 1d, 0d, false, Setting.Scope.CLUSTER);
-    public static final Setting<Double> WEIGHT_LONG_SETTING  = Setting.doubleSetting("cache.recycler.page.weight.longs", 1d, 0d, false, Setting.Scope.CLUSTER);
-    public static final Setting<Double> WEIGHT_INT_SETTING  = Setting.doubleSetting("cache.recycler.page.weight.ints", 1d, 0d, false, Setting.Scope.CLUSTER);
+    public static final Setting<Type> TYPE_SETTING =
+        new Setting<>("cache.recycler.page.type", Type.CONCURRENT.name(), Type::parse, Property.NodeScope);
+    public static final Setting<ByteSizeValue> LIMIT_HEAP_SETTING  =
+        Setting.byteSizeSetting("cache.recycler.page.limit.heap", "10%", Property.NodeScope);
+    public static final Setting<Double> WEIGHT_BYTES_SETTING  =
+        Setting.doubleSetting("cache.recycler.page.weight.bytes", 1d, 0d, Property.NodeScope);
+    public static final Setting<Double> WEIGHT_LONG_SETTING  =
+        Setting.doubleSetting("cache.recycler.page.weight.longs", 1d, 0d, Property.NodeScope);
+    public static final Setting<Double> WEIGHT_INT_SETTING  =
+        Setting.doubleSetting("cache.recycler.page.weight.ints", 1d, 0d, Property.NodeScope);
     // object pages are less useful to us so we give them a lower weight by default
-    public static final Setting<Double> WEIGHT_OBJECTS_SETTING  = Setting.doubleSetting("cache.recycler.page.weight.objects", 0.1d, 0d, false, Setting.Scope.CLUSTER);
+    public static final Setting<Double> WEIGHT_OBJECTS_SETTING  =
+        Setting.doubleSetting("cache.recycler.page.weight.objects", 0.1d, 0d, Property.NodeScope);
 
     private final Recycler<byte[]> bytePage;
     private final Recycler<int[]> intPage;
diff --git a/core/src/main/java/org/elasticsearch/cli/Command.java b/core/src/main/java/org/elasticsearch/cli/Command.java
new file mode 100644
index 00000000000..9e6afdd6638
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cli/Command.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cli;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import joptsimple.OptionException;
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.elasticsearch.common.SuppressForbidden;
+
+/**
+ * An action to execute within a cli.
+ */
+public abstract class Command {
+
+    /** A description of the command, used in the help output. */
+    protected final String description;
+
+    /** The option parser for this command. */
+    protected final OptionParser parser = new OptionParser();
+
+    private final OptionSpec<Void> helpOption = parser.acceptsAll(Arrays.asList("h", "help"), "show help").forHelp();
+    private final OptionSpec<Void> silentOption = parser.acceptsAll(Arrays.asList("s", "silent"), "show minimal output");
+    private final OptionSpec<Void> verboseOption = parser.acceptsAll(Arrays.asList("v", "verbose"), "show verbose output");
+
+    public Command(String description) {
+        this.description = description;
+    }
+
+    /** Parses options for this command from args and executes it. */
+    public final int main(String[] args, Terminal terminal) throws Exception {
+        try {
+            mainWithoutErrorHandling(args, terminal);
+        } catch (OptionException e) {
+            printHelp(terminal);
+            terminal.println(Terminal.Verbosity.SILENT, "ERROR: " + e.getMessage());
+            return ExitCodes.USAGE;
+        } catch (UserError e) {
+            terminal.println(Terminal.Verbosity.SILENT, "ERROR: " + e.getMessage());
+            return e.exitCode;
+        }
+        return ExitCodes.OK;
+    }
+
+    /**
+     * Executes the command, but all errors are thrown.
+     */
+    void mainWithoutErrorHandling(String[] args, Terminal terminal) throws Exception {
+        final OptionSet options = parser.parse(args);
+
+        if (options.has(helpOption)) {
+            printHelp(terminal);
+            return;
+        }
+
+        if (options.has(silentOption)) {
+            if (options.has(verboseOption)) {
+                // mutually exclusive, we can remove this with jopt-simple 5.0, which natively supports it
+                throw new UserError(ExitCodes.USAGE, "Cannot specify -s and -v together");
+            }
+            terminal.setVerbosity(Terminal.Verbosity.SILENT);
+        } else if (options.has(verboseOption)) {
+            terminal.setVerbosity(Terminal.Verbosity.VERBOSE);
+        } else {
+            terminal.setVerbosity(Terminal.Verbosity.NORMAL);
+        }
+
+        execute(terminal, options);
+    }
+
+    /** Prints a help message for the command to the terminal. */
+    private void printHelp(Terminal terminal) throws IOException {
+        terminal.println(description);
+        terminal.println("");
+        printAdditionalHelp(terminal);
+        parser.printHelpOn(terminal.getWriter());
+    }
+
+    /** Prints additional help information, specific to the command */
+    protected void printAdditionalHelp(Terminal terminal) {}
+
+    @SuppressForbidden(reason = "Allowed to exit explicitly from #main()")
+    protected static void exit(int status) {
+        System.exit(status);
+    }
+
+    /**
+     * Executes this command.
+     *
+     * Any runtime user errors (like an input file that does not exist), should throw a {@link UserError}. */
+    protected abstract void execute(Terminal terminal, OptionSet options) throws Exception;
+}
diff --git a/core/src/main/java/org/elasticsearch/cli/ExitCodes.java b/core/src/main/java/org/elasticsearch/cli/ExitCodes.java
new file mode 100644
index 00000000000..d08deb8b1ad
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cli/ExitCodes.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cli;
+
+/**
+ * POSIX exit codes.
+ */
+public class ExitCodes {
+    public static final int OK = 0;
+    public static final int USAGE = 64;          /* command line usage error */
+    public static final int DATA_ERROR = 65;     /* data format error */
+    public static final int NO_INPUT = 66;       /* cannot open input */
+    public static final int NO_USER = 67;        /* addressee unknown */
+    public static final int NO_HOST = 68;        /* host name unknown */
+    public static final int UNAVAILABLE = 69;    /* service unavailable */
+    public static final int CODE_ERROR = 70;     /* internal software error */
+    public static final int CANT_CREATE = 73;    /* can't create (user) output file */
+    public static final int IO_ERROR = 74;       /* input/output error */
+    public static final int TEMP_FAILURE = 75;   /* temp failure; user is invited to retry */
+    public static final int PROTOCOL = 76;       /* remote error in protocol */
+    public static final int NOPERM = 77;         /* permission denied */
+    public static final int CONFIG = 78;         /* configuration error */
+
+    private ExitCodes() { /* no instance, just constants */ }
+}
diff --git a/core/src/main/java/org/elasticsearch/cli/MultiCommand.java b/core/src/main/java/org/elasticsearch/cli/MultiCommand.java
new file mode 100644
index 00000000000..a9feee0c9bf
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cli/MultiCommand.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cli;
+
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import joptsimple.NonOptionArgumentSpec;
+import joptsimple.OptionSet;
+
+/**
+ * A cli tool which is made up of multiple subcommands.
+ */
+public class MultiCommand extends Command {
+
+    protected final Map<String, Command> subcommands = new LinkedHashMap<>();
+
+    private final NonOptionArgumentSpec<String> arguments = parser.nonOptions("command");
+
+    public MultiCommand(String description) {
+        super(description);
+        parser.posixlyCorrect(true);
+    }
+
+    @Override
+    protected void printAdditionalHelp(Terminal terminal) {
+        if (subcommands.isEmpty()) {
+            throw new IllegalStateException("No subcommands configured");
+        }
+        terminal.println("Commands");
+        terminal.println("--------");
+        for (Map.Entry<String, Command> subcommand : subcommands.entrySet()) {
+            terminal.println(subcommand.getKey() + " - " + subcommand.getValue().description);
+        }
+        terminal.println("");
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options) throws Exception {
+        if (subcommands.isEmpty()) {
+            throw new IllegalStateException("No subcommands configured");
+        }
+        String[] args = arguments.values(options).toArray(new String[0]);
+        if (args.length == 0) {
+            throw new UserError(ExitCodes.USAGE, "Missing command");
+        }
+        Command subcommand = subcommands.get(args[0]);
+        if (subcommand == null) {
+            throw new UserError(ExitCodes.USAGE, "Unknown command [" + args[0] + "]");
+        }
+        subcommand.mainWithoutErrorHandling(Arrays.copyOfRange(args, 1, args.length), terminal);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/cli/Terminal.java b/core/src/main/java/org/elasticsearch/cli/Terminal.java
similarity index 91%
rename from core/src/main/java/org/elasticsearch/common/cli/Terminal.java
rename to core/src/main/java/org/elasticsearch/cli/Terminal.java
index fbef1f78cc3..d2dc57263dc 100644
--- a/core/src/main/java/org/elasticsearch/common/cli/Terminal.java
+++ b/core/src/main/java/org/elasticsearch/cli/Terminal.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.common.cli;
+package org.elasticsearch.cli;
 
 import java.io.BufferedReader;
 import java.io.Console;
@@ -29,7 +29,7 @@ import java.nio.charset.Charset;
 import org.elasticsearch.common.SuppressForbidden;
 
 /**
- * A Terminal wraps access to reading input and writing output for a {@link CliTool}.
+ * A Terminal wraps access to reading input and writing output for a cli.
  *
  * The available methods are similar to those of {@link Console}, with the ability
  * to read either normal text or a password, and the ability to print a line
@@ -61,7 +61,7 @@ public abstract class Terminal {
     }
 
     /** Sets the verbosity of the terminal. */
-    void setVerbosity(Verbosity verbosity) {
+    public void setVerbosity(Verbosity verbosity) {
         this.verbosity = verbosity;
     }
 
@@ -89,35 +89,35 @@ public abstract class Terminal {
 
     private static class ConsoleTerminal extends Terminal {
 
-        private static final Console console = System.console();
+        private static final Console CONSOLE = System.console();
 
         ConsoleTerminal() {
             super(System.lineSeparator());
         }
 
         static boolean isSupported() {
-            return console != null;
+            return CONSOLE != null;
         }
 
         @Override
         public PrintWriter getWriter() {
-            return console.writer();
+            return CONSOLE.writer();
         }
 
         @Override
         public String readText(String prompt) {
-            return console.readLine("%s", prompt);
+            return CONSOLE.readLine("%s", prompt);
         }
 
         @Override
         public char[] readSecret(String prompt) {
-            return console.readPassword("%s", prompt);
+            return CONSOLE.readPassword("%s", prompt);
         }
     }
 
     private static class SystemTerminal extends Terminal {
 
-        private final PrintWriter writer = newWriter();
+        private static final PrintWriter WRITER = newWriter();
 
         SystemTerminal() {
             super(System.lineSeparator());
@@ -130,7 +130,7 @@ public abstract class Terminal {
 
         @Override
         public PrintWriter getWriter() {
-            return writer;
+            return WRITER;
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/common/cli/UserError.java b/core/src/main/java/org/elasticsearch/cli/UserError.java
similarity index 79%
rename from core/src/main/java/org/elasticsearch/common/cli/UserError.java
rename to core/src/main/java/org/elasticsearch/cli/UserError.java
index ad709830885..2a4f2bf1233 100644
--- a/core/src/main/java/org/elasticsearch/common/cli/UserError.java
+++ b/core/src/main/java/org/elasticsearch/cli/UserError.java
@@ -17,19 +17,19 @@
  * under the License.
  */
 
-package org.elasticsearch.common.cli;
+package org.elasticsearch.cli;
 
 /**
- * An exception representing a user fixable problem in {@link CliTool} usage.
+ * An exception representing a user fixable problem in {@link Command} usage.
  */
 public class UserError extends Exception {
 
     /** The exist status the cli should use when catching this user error. */
-    public final CliTool.ExitStatus exitStatus;
+    public final int exitCode;
 
     /** Constructs a UserError with an exit status and message to show the user. */
-    public UserError(CliTool.ExitStatus exitStatus, String msg) {
+    public UserError(int exitCode, String msg) {
         super(msg);
-        this.exitStatus = exitStatus;
+        this.exitCode = exitCode;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/client/Client.java b/core/src/main/java/org/elasticsearch/client/Client.java
index f81ba9eb1b1..1c7676bc91c 100644
--- a/core/src/main/java/org/elasticsearch/client/Client.java
+++ b/core/src/main/java/org/elasticsearch/client/Client.java
@@ -19,12 +19,8 @@
 
 package org.elasticsearch.client;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestBuilder;
-import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
@@ -72,9 +68,6 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.action.search.SearchScrollRequestBuilder;
-import org.elasticsearch.action.suggest.SuggestRequest;
-import org.elasticsearch.action.suggest.SuggestRequestBuilder;
-import org.elasticsearch.action.suggest.SuggestResponse;
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequestBuilder;
 import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
@@ -87,6 +80,7 @@ import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 import java.util.Map;
@@ -114,7 +108,7 @@ public interface Client extends ElasticsearchClient, Releasable {
             default:
                 throw new IllegalArgumentException("Can't parse [client.type] must be one of [node, transport]");
         }
-    }, false, Setting.Scope.CLUSTER);
+    }, Property.NodeScope);
 
     /**
      * The admin client that can be used to perform administrative operations.
@@ -370,29 +364,6 @@ public interface Client extends ElasticsearchClient, Releasable {
      */
     MultiGetRequestBuilder prepareMultiGet();
 
-    /**
-     * Suggestion matching a specific phrase.
-     *
-     * @param request The suggest request
-     * @return The result future
-     * @see Requests#suggestRequest(String...)
-     */
-    ActionFuture<SuggestResponse> suggest(SuggestRequest request);
-
-    /**
-     * Suggestions matching a specific phrase.
-     *
-     * @param request  The suggest request
-     * @param listener A listener to be notified of the result
-     * @see Requests#suggestRequest(String...)
-     */
-    void suggest(SuggestRequest request, ActionListener<SuggestResponse> listener);
-
-    /**
-     * Suggestions matching a specific phrase.
-     */
-    SuggestRequestBuilder prepareSuggest(String... indices);
-
     /**
      * Search across one or more indices and one or more types with a query.
      *
diff --git a/core/src/main/java/org/elasticsearch/client/Requests.java b/core/src/main/java/org/elasticsearch/client/Requests.java
index 3cf4f3dc6cb..276bd9d9062 100644
--- a/core/src/main/java/org/elasticsearch/client/Requests.java
+++ b/core/src/main/java/org/elasticsearch/client/Requests.java
@@ -60,7 +60,6 @@ import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchScrollRequest;
-import org.elasticsearch.action.suggest.SuggestRequest;
 import org.elasticsearch.common.xcontent.XContentType;
 
 /**
@@ -125,16 +124,6 @@ public class Requests {
         return new GetRequest(index);
     }
 
-    /**
-     * Creates a suggest request for getting suggestions from provided <code>indices</code>.
-     * The suggest query has to be set using the JSON source using {@link org.elasticsearch.action.suggest.SuggestRequest#suggest(org.elasticsearch.common.bytes.BytesReference)}.
-     * @param indices The indices to suggest from. Use <tt>null</tt> or <tt>_all</tt> to execute against all indices
-     * @see org.elasticsearch.client.Client#suggest(org.elasticsearch.action.suggest.SuggestRequest)
-     */
-    public static SuggestRequest suggestRequest(String... indices) {
-        return new SuggestRequest(indices);
-    }
-
     /**
      * Creates a search request against one or more indices. Note, the search source must be set either using the
      * actual JSON search source, or the {@link org.elasticsearch.search.builder.SearchSourceBuilder}.
@@ -342,7 +331,8 @@ public class Requests {
     /**
      * Creates a cluster health request.
      *
-     * @param indices The indices to provide additional cluster health information for. Use <tt>null</tt> or <tt>_all</tt> to execute against all indices
+     * @param indices The indices to provide additional cluster health information for.
+     *                Use <tt>null</tt> or <tt>_all</tt> to execute against all indices
      * @return The cluster health request
      * @see org.elasticsearch.client.ClusterAdminClient#health(org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest)
      */
diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
index f729d5287df..0044890ee35 100644
--- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
+++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
@@ -314,10 +314,6 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollAction;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.action.search.SearchScrollRequestBuilder;
-import org.elasticsearch.action.suggest.SuggestAction;
-import org.elasticsearch.action.suggest.SuggestRequest;
-import org.elasticsearch.action.suggest.SuggestRequestBuilder;
-import org.elasticsearch.action.suggest.SuggestResponse;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.action.termvectors.MultiTermVectorsAction;
@@ -660,21 +656,6 @@ public abstract class AbstractClient extends AbstractComponent implements Client
         return new MultiSearchRequestBuilder(this, MultiSearchAction.INSTANCE);
     }
 
-    @Override
-    public ActionFuture<SuggestResponse> suggest(final SuggestRequest request) {
-        return execute(SuggestAction.INSTANCE, request);
-    }
-
-    @Override
-    public void suggest(final SuggestRequest request, final ActionListener<SuggestResponse> listener) {
-        execute(SuggestAction.INSTANCE, request, listener);
-    }
-
-    @Override
-    public SuggestRequestBuilder prepareSuggest(String... indices) {
-        return new SuggestRequestBuilder(this, SuggestAction.INSTANCE).setIndices(indices);
-    }
-
     @Override
     public ActionFuture<TermVectorsResponse> termVectors(final TermVectorsRequest request) {
         return execute(TermVectorsAction.INSTANCE, request);
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
index 1e605b9de06..abbc5823b2a 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
@@ -34,6 +34,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
@@ -100,10 +101,14 @@ public class TransportClientNodesService extends AbstractComponent {
     private volatile boolean closed;
 
 
-    public static final Setting<TimeValue> CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL = Setting.positiveTimeSetting("client.transport.nodes_sampler_interval", timeValueSeconds(5), false, Setting.Scope.CLUSTER);
-    public static final Setting<TimeValue> CLIENT_TRANSPORT_PING_TIMEOUT = Setting.positiveTimeSetting("client.transport.ping_timeout", timeValueSeconds(5), false, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> CLIENT_TRANSPORT_IGNORE_CLUSTER_NAME = Setting.boolSetting("client.transport.ignore_cluster_name", false, false, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> CLIENT_TRANSPORT_SNIFF = Setting.boolSetting("client.transport.sniff", false, false, Setting.Scope.CLUSTER);
+    public static final Setting<TimeValue> CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL =
+        Setting.positiveTimeSetting("client.transport.nodes_sampler_interval", timeValueSeconds(5), Property.NodeScope);
+    public static final Setting<TimeValue> CLIENT_TRANSPORT_PING_TIMEOUT =
+        Setting.positiveTimeSetting("client.transport.ping_timeout", timeValueSeconds(5), Property.NodeScope);
+    public static final Setting<Boolean> CLIENT_TRANSPORT_IGNORE_CLUSTER_NAME =
+        Setting.boolSetting("client.transport.ignore_cluster_name", false, Property.NodeScope);
+    public static final Setting<Boolean> CLIENT_TRANSPORT_SNIFF =
+        Setting.boolSetting("client.transport.sniff", false, Property.NodeScope);
 
     @Inject
     public TransportClientNodesService(Settings settings, ClusterName clusterName, TransportService transportService,
@@ -119,7 +124,7 @@ public class TransportClientNodesService extends AbstractComponent {
         this.ignoreClusterName = CLIENT_TRANSPORT_IGNORE_CLUSTER_NAME.get(this.settings);
 
         if (logger.isDebugEnabled()) {
-            logger.debug("node_sampler_interval[" + nodesSamplerInterval + "]");
+            logger.debug("node_sampler_interval[{}]", nodesSamplerInterval);
         }
 
         if (CLIENT_TRANSPORT_SNIFF.get(this.settings)) {
@@ -318,7 +323,7 @@ public class TransportClientNodesService extends AbstractComponent {
                         transportService.connectToNode(node);
                     } catch (Throwable e) {
                         it.remove();
-                        logger.debug("failed to connect to discovered node [" + node + "]", e);
+                        logger.debug("failed to connect to discovered node [{}]", e, node);
                     }
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java b/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java
index e851b7814da..98853e8447f 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java
@@ -23,6 +23,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.index.Index;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -120,7 +121,7 @@ public class ClusterChangedEvent {
     /**
      * Returns the indices deleted in this event
      */
-    public List<String> indicesDeleted() {
+    public List<Index> indicesDeleted() {
         // If the new cluster state has a new cluster UUID, the likely scenario is that a node was elected
         // master that has had its data directory wiped out, in which case we don't want to delete the indices and lose data;
         // rather we want to import them as dangling indices instead.  So we check here if the cluster UUID differs from the previous
@@ -131,17 +132,18 @@ public class ClusterChangedEvent {
         if (metaDataChanged() == false || isNewCluster()) {
             return Collections.emptyList();
         }
-        List<String> deleted = null;
-        for (ObjectCursor<String> cursor : previousState.metaData().indices().keys()) {
-            String index = cursor.value;
-            if (!state.metaData().hasIndex(index)) {
+        List<Index> deleted = null;
+        for (ObjectCursor<IndexMetaData> cursor : previousState.metaData().indices().values()) {
+            IndexMetaData index = cursor.value;
+            IndexMetaData current = state.metaData().index(index.getIndex());
+            if (current == null) {
                 if (deleted == null) {
                     deleted = new ArrayList<>();
                 }
-                deleted.add(index);
+                deleted.add(index.getIndex());
             }
         }
-        return deleted == null ? Collections.<String>emptyList() : deleted;
+        return deleted == null ? Collections.<Index>emptyList() : deleted;
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index 3e668191ff3..47dd2ce9ae6 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -53,11 +53,12 @@ import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationD
 import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
-import org.elasticsearch.cluster.service.InternalClusterService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.ExtensionPoint;
 import org.elasticsearch.gateway.GatewayAllocator;
@@ -74,7 +75,8 @@ public class ClusterModule extends AbstractModule {
 
     public static final String EVEN_SHARD_COUNT_ALLOCATOR = "even_shard";
     public static final String BALANCED_ALLOCATOR = "balanced"; // default
-    public static final Setting<String> SHARDS_ALLOCATOR_TYPE_SETTING = new Setting<>("cluster.routing.allocation.type", BALANCED_ALLOCATOR, Function.identity(), false, Setting.Scope.CLUSTER);
+    public static final Setting<String> SHARDS_ALLOCATOR_TYPE_SETTING =
+        new Setting<>("cluster.routing.allocation.type", BALANCED_ALLOCATOR, Function.identity(), Property.NodeScope);
     public static final List<Class<? extends AllocationDecider>> DEFAULT_ALLOCATION_DECIDERS =
         Collections.unmodifiableList(Arrays.asList(
             SameShardAllocationDecider.class,
@@ -135,7 +137,8 @@ public class ClusterModule extends AbstractModule {
         bind(GatewayAllocator.class).asEagerSingleton();
         bind(AllocationService.class).asEagerSingleton();
         bind(DiscoveryNodeService.class).asEagerSingleton();
-        bind(ClusterService.class).to(InternalClusterService.class).asEagerSingleton();
+        bind(ClusterService.class).asEagerSingleton();
+        bind(NodeConnectionsService.class).asEagerSingleton();
         bind(OperationRouting.class).asEagerSingleton();
         bind(MetaDataCreateIndexService.class).asEagerSingleton();
         bind(MetaDataDeleteIndexService.class).asEagerSingleton();
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterName.java b/core/src/main/java/org/elasticsearch/cluster/ClusterName.java
index daf3000d710..09c64065dbd 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterName.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterName.java
@@ -23,6 +23,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 import java.io.IOException;
@@ -37,7 +38,7 @@ public class ClusterName implements Streamable {
             throw new IllegalArgumentException("[cluster.name] must not be empty");
         }
         return s;
-    }, false, Setting.Scope.CLUSTER);
+    }, Property.NodeScope);
 
 
     public static final ClusterName DEFAULT = new ClusterName(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY).intern());
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterService.java b/core/src/main/java/org/elasticsearch/cluster/ClusterService.java
deleted file mode 100644
index 27df4b9e96f..00000000000
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterService.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster;
-
-import org.elasticsearch.cluster.block.ClusterBlock;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.OperationRouting;
-import org.elasticsearch.cluster.service.PendingClusterTask;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.component.LifecycleComponent;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.tasks.TaskManager;
-
-import java.util.List;
-
-/**
- * The cluster service allowing to both register for cluster state events ({@link ClusterStateListener})
- * and submit state update tasks ({@link ClusterStateUpdateTask}.
- */
-public interface ClusterService extends LifecycleComponent<ClusterService> {
-
-    /**
-     * The local node.
-     */
-    DiscoveryNode localNode();
-
-    /**
-     * The current state.
-     */
-    ClusterState state();
-
-    /**
-     * Adds an initial block to be set on the first cluster state created.
-     */
-    void addInitialStateBlock(ClusterBlock block) throws IllegalStateException;
-
-    /**
-     * Remove an initial block to be set on the first cluster state created.
-     */
-    void removeInitialStateBlock(ClusterBlock block) throws IllegalStateException;
-
-    /**
-     * Remove an initial block to be set on the first cluster state created.
-     */
-    void removeInitialStateBlock(int blockId) throws IllegalStateException;
-
-    /**
-     * The operation routing.
-     */
-    OperationRouting operationRouting();
-
-    /**
-     * Adds a priority listener for updated cluster states.
-     */
-    void addFirst(ClusterStateListener listener);
-
-    /**
-     * Adds last listener.
-     */
-    void addLast(ClusterStateListener listener);
-
-    /**
-     * Adds a listener for updated cluster states.
-     */
-    void add(ClusterStateListener listener);
-
-    /**
-     * Removes a listener for updated cluster states.
-     */
-    void remove(ClusterStateListener listener);
-
-    /**
-     * Add a listener for on/off local node master events
-     */
-    void add(LocalNodeMasterListener listener);
-
-    /**
-     * Remove the given listener for on/off local master events
-     */
-    void remove(LocalNodeMasterListener listener);
-
-    /**
-     * Adds a cluster state listener that will timeout after the provided timeout,
-     * and is executed after the clusterstate has been successfully applied ie. is
-     * in state {@link org.elasticsearch.cluster.ClusterState.ClusterStateStatus#APPLIED}
-     * NOTE: a {@code null} timeout means that the listener will never be removed
-     * automatically
-     */
-    void add(@Nullable TimeValue timeout, TimeoutClusterStateListener listener);
-
-    /**
-     * Submits a cluster state update task; submitted updates will be
-     * batched across the same instance of executor. The exact batching
-     * semantics depend on the underlying implementation but a rough
-     * guideline is that if the update task is submitted while there
-     * are pending update tasks for the same executor, these update
-     * tasks will all be executed on the executor in a single batch
-     *
-     * @param source   the source of the cluster state update task
-     * @param task     the state needed for the cluster state update task
-     * @param config   the cluster state update task configuration
-     * @param executor the cluster state update task executor; tasks
-     *                 that share the same executor will be executed
-     *                 batches on this executor
-     * @param listener callback after the cluster state update task
-     *                 completes
-     * @param <T>      the type of the cluster state update task state
-     */
-    <T> void submitStateUpdateTask(final String source, final T task,
-                                   final ClusterStateTaskConfig config,
-                                   final ClusterStateTaskExecutor<T> executor,
-                                   final ClusterStateTaskListener listener);
-
-    /**
-     * Submits a cluster state update task; unlike {@link #submitStateUpdateTask(String, Object, ClusterStateTaskConfig, ClusterStateTaskExecutor, ClusterStateTaskListener)},
-     * submitted updates will not be batched.
-     *
-     * @param source     the source of the cluster state update task
-     * @param updateTask the full context for the cluster state update
-     *                   task
-     */
-    void submitStateUpdateTask(final String source, final ClusterStateUpdateTask updateTask);
-
-    /**
-     * Returns the tasks that are pending.
-     */
-    List<PendingClusterTask> pendingTasks();
-
-    /**
-     * Returns the number of currently pending tasks.
-     */
-    int numberOfPendingTasks();
-
-    /**
-     * Returns the maximum wait time for tasks in the queue
-     *
-     * @return A zero time value if the queue is empty, otherwise the time value oldest task waiting in the queue
-     */
-    TimeValue getMaxTaskWaitTime();
-
-    /**
-     * Returns task manager created in the cluster service
-     */
-    TaskManager getTaskManager();
-}
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
index 1b3ddcfebf9..1ac379555ab 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
@@ -37,7 +37,7 @@ import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.cluster.service.InternalClusterService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -63,18 +63,18 @@ import java.util.Set;
 
 /**
  * Represents the current state of the cluster.
- *
+ * <p>
  * The cluster state object is immutable with an
  * exception of the {@link RoutingNodes} structure, which is built on demand from the {@link RoutingTable},
  * and cluster state {@link #status}, which is updated during cluster state publishing and applying
  * processing.  The cluster state can be updated only on the master node. All updates are performed by on a
- * single thread and controlled by the {@link InternalClusterService}. After every update the
+ * single thread and controlled by the {@link ClusterService}. After every update the
  * {@link Discovery#publish} method publishes new version of the cluster state to all other nodes in the
  * cluster.  The actual publishing mechanism is delegated to the {@link Discovery#publish} method and depends on
  * the type of discovery. For example, for local discovery it is implemented by the {@link LocalDiscovery#publish}
  * method. In the Zen Discovery it is handled in the {@link PublishClusterStateAction#publish} method. The
  * publishing mechanism can be overridden by other discovery.
- *
+ * <p>
  * The cluster state implements the {@link Diffable} interface in order to support publishing of cluster state
  * differences instead of the entire state on each change. The publishing mechanism should only send differences
  * to a node if this node was present in the previous version of the cluster state. If a node is not present was
@@ -135,7 +135,7 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
 
     public static <T extends Custom> T lookupPrototypeSafe(String type) {
         @SuppressWarnings("unchecked")
-        T proto = (T)customPrototypes.get(type);
+        T proto = (T) customPrototypes.get(type);
         if (proto == null) {
             throw new IllegalArgumentException("No custom state prototype registered for type [" + type + "], node likely missing plugins");
         }
@@ -281,6 +281,16 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
         sb.append("state uuid: ").append(stateUUID).append("\n");
         sb.append("from_diff: ").append(wasReadFromDiff).append("\n");
         sb.append("meta data version: ").append(metaData.version()).append("\n");
+        for (IndexMetaData indexMetaData : metaData) {
+            final String TAB = "   ";
+            sb.append(TAB).append(indexMetaData.getIndex());
+            sb.append(": v[").append(indexMetaData.getVersion()).append("]\n");
+            for (int shard = 0; shard < indexMetaData.getNumberOfShards(); shard++) {
+                sb.append(TAB).append(TAB).append(shard).append(": ");
+                sb.append("p_term [").append(indexMetaData.primaryTerm(shard)).append("], ");
+                sb.append("a_ids ").append(indexMetaData.activeAllocationIds(shard)).append("\n");
+            }
+        }
         sb.append(blocks().prettyPrint());
         sb.append(nodes().prettyPrint());
         sb.append(routingTable().prettyPrint());
@@ -477,6 +487,12 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
                 }
                 builder.endArray();
 
+                builder.startObject(IndexMetaData.KEY_PRIMARY_TERMS);
+                for (int shard = 0; shard < indexMetaData.getNumberOfShards(); shard++) {
+                    builder.field(Integer.toString(shard), indexMetaData.primaryTerm(shard));
+                }
+                builder.endObject();
+
                 builder.startObject(IndexMetaData.KEY_ACTIVE_ALLOCATIONS);
                 for (IntObjectCursor<Set<String>> cursor : indexMetaData.getActiveAllocationIds()) {
                     builder.startArray(String.valueOf(cursor.key));
@@ -487,6 +503,7 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
                 }
                 builder.endObject();
 
+                // index metadata
                 builder.endObject();
             }
             builder.endObject();
@@ -683,16 +700,16 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
         }
 
         /**
-         * @param data               input bytes
-         * @param localNode          used to set the local node in the cluster state.
+         * @param data      input bytes
+         * @param localNode used to set the local node in the cluster state.
          */
         public static ClusterState fromBytes(byte[] data, DiscoveryNode localNode) throws IOException {
             return readFrom(StreamInput.wrap(data), localNode);
         }
 
         /**
-         * @param in                 input stream
-         * @param localNode          used to set the local node in the cluster state. can be null.
+         * @param in        input stream
+         * @param localNode used to set the local node in the cluster state. can be null.
          */
         public static ClusterState readFrom(StreamInput in, @Nullable DiscoveryNode localNode) throws IOException {
             return PROTO.readFrom(in, localNode);
@@ -791,17 +808,17 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
             metaData = proto.metaData.readDiffFrom(in);
             blocks = proto.blocks.readDiffFrom(in);
             customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(),
-                    new DiffableUtils.DiffableValueSerializer<String, Custom>() {
-                @Override
-                public Custom read(StreamInput in, String key) throws IOException {
-                    return lookupPrototypeSafe(key).readFrom(in);
-                }
+                new DiffableUtils.DiffableValueSerializer<String, Custom>() {
+                    @Override
+                    public Custom read(StreamInput in, String key) throws IOException {
+                        return lookupPrototypeSafe(key).readFrom(in);
+                    }
 
-                @Override
-                public Diff<Custom> readDiff(StreamInput in, String key) throws IOException {
-                    return lookupPrototypeSafe(key).readDiffFrom(in);
-                }
-            });
+                    @Override
+                    public Diff<Custom> readDiff(StreamInput in, String key) throws IOException {
+                        return lookupPrototypeSafe(key).readDiffFrom(in);
+                    }
+                });
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java
index dd30a711688..d79a00dc3fe 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.cluster;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.unit.TimeValue;
@@ -44,7 +45,7 @@ public class ClusterStateObserver {
         }
     };
 
-    private final  ClusterService clusterService;
+    private final ClusterService clusterService;
     private final ThreadContext contextHolder;
     volatile TimeValue timeOutValue;
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
index 5107b4495ab..9a9ee06ce19 100644
--- a/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
@@ -33,12 +33,14 @@ import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
@@ -64,8 +66,12 @@ import java.util.concurrent.TimeUnit;
  */
 public class InternalClusterInfoService extends AbstractComponent implements ClusterInfoService, LocalNodeMasterListener, ClusterStateListener {
 
-    public static final Setting<TimeValue> INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING = Setting.timeSetting("cluster.info.update.interval", TimeValue.timeValueSeconds(30), TimeValue.timeValueSeconds(10), true, Setting.Scope.CLUSTER);
-    public static final Setting<TimeValue> INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING = Setting.positiveTimeSetting("cluster.info.update.timeout", TimeValue.timeValueSeconds(15), true, Setting.Scope.CLUSTER);
+    public static final Setting<TimeValue> INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING =
+        Setting.timeSetting("cluster.info.update.interval", TimeValue.timeValueSeconds(30), TimeValue.timeValueSeconds(10),
+            Property.Dynamic, Property.NodeScope);
+    public static final Setting<TimeValue> INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING =
+        Setting.positiveTimeSetting("cluster.info.update.timeout", TimeValue.timeValueSeconds(15),
+            Property.Dynamic, Property.NodeScope);
 
     private volatile TimeValue updateFrequency;
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java b/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
new file mode 100644
index 00000000000..698f9d1090c
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.util.concurrent.FutureUtils;
+import org.elasticsearch.common.util.concurrent.KeyedLock;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ScheduledFuture;
+
+import static org.elasticsearch.common.settings.Setting.Property;
+import static org.elasticsearch.common.settings.Setting.positiveTimeSetting;
+
+
+/**
+ * This component is responsible for connecting to nodes once they are added to the cluster state, and disconnect when they are
+ * removed. Also, it periodically checks that all connections are still open and if needed restores them.
+ * Note that this component is *not* responsible for removing nodes from the cluster if they disconnect / do not respond
+ * to pings. This is done by {@link org.elasticsearch.discovery.zen.fd.NodesFaultDetection}. Master fault detection
+ * is done by {@link org.elasticsearch.discovery.zen.fd.MasterFaultDetection}.
+ */
+public class NodeConnectionsService extends AbstractLifecycleComponent<NodeConnectionsService> {
+
+    public static final Setting<TimeValue> CLUSTER_NODE_RECONNECT_INTERVAL_SETTING =
+            positiveTimeSetting("cluster.nodes.reconnect_interval", TimeValue.timeValueSeconds(10), Property.NodeScope);
+    private final ThreadPool threadPool;
+    private final TransportService transportService;
+
+    // map between current node and the number of failed connection attempts. 0 means successfully connected.
+    // if a node doesn't appear in this list it shouldn't be monitored
+    private ConcurrentMap<DiscoveryNode, Integer> nodes = ConcurrentCollections.newConcurrentMap();
+
+    final private KeyedLock<DiscoveryNode> nodeLocks = new KeyedLock<>();
+
+    private final TimeValue reconnectInterval;
+
+    private volatile ScheduledFuture<?> backgroundFuture = null;
+
+    @Inject
+    public NodeConnectionsService(Settings settings, ThreadPool threadPool, TransportService transportService) {
+        super(settings);
+        this.threadPool = threadPool;
+        this.transportService = transportService;
+        this.reconnectInterval = NodeConnectionsService.CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.get(settings);
+    }
+
+    public void connectToAddedNodes(ClusterChangedEvent event) {
+
+        // TODO: do this in parallel (and wait)
+        for (final DiscoveryNode node : event.nodesDelta().addedNodes()) {
+            try (Releasable ignored = nodeLocks.acquire(node)) {
+                Integer current = nodes.put(node, 0);
+                assert current == null : "node " + node + " was added in event but already in internal nodes";
+                validateNodeConnected(node);
+            }
+        }
+    }
+
+    public void disconnectFromRemovedNodes(ClusterChangedEvent event) {
+        for (final DiscoveryNode node : event.nodesDelta().removedNodes()) {
+            try (Releasable ignored = nodeLocks.acquire(node)) {
+                Integer current = nodes.remove(node);
+                assert current != null : "node " + node + " was removed in event but not in internal nodes";
+                try {
+                    transportService.disconnectFromNode(node);
+                } catch (Throwable e) {
+                    logger.warn("failed to disconnect to node [" + node + "]", e);
+                }
+            }
+        }
+    }
+
+    void validateNodeConnected(DiscoveryNode node) {
+        assert nodeLocks.isHeldByCurrentThread(node) : "validateNodeConnected must be called under lock";
+        if (lifecycle.stoppedOrClosed() ||
+                nodes.containsKey(node) == false) { // we double check existence of node since connectToNode might take time...
+            // nothing to do
+        } else {
+            try {
+                // connecting to an already connected node is a noop
+                transportService.connectToNode(node);
+                nodes.put(node, 0);
+            } catch (Exception e) {
+                Integer nodeFailureCount = nodes.get(node);
+                assert nodeFailureCount != null : node + " didn't have a counter in nodes map";
+                nodeFailureCount = nodeFailureCount + 1;
+                // log every 6th failure
+                if ((nodeFailureCount % 6) == 1) {
+                    logger.warn("failed to connect to node {} (tried [{}] times)", e, node, nodeFailureCount);
+                }
+                nodes.put(node, nodeFailureCount);
+            }
+        }
+    }
+
+    class ConnectionChecker extends AbstractRunnable {
+
+        @Override
+        public void onFailure(Throwable t) {
+            logger.warn("unexpected error while checking for node reconnects", t);
+        }
+
+        protected void doRun() {
+            for (DiscoveryNode node : nodes.keySet()) {
+                try (Releasable ignored = nodeLocks.acquire(node)) {
+                    validateNodeConnected(node);
+                }
+            }
+        }
+
+        @Override
+        public void onAfter() {
+            if (lifecycle.started()) {
+                backgroundFuture = threadPool.schedule(reconnectInterval, ThreadPool.Names.GENERIC, this);
+            }
+        }
+    }
+
+    @Override
+    protected void doStart() {
+        backgroundFuture = threadPool.schedule(reconnectInterval, ThreadPool.Names.GENERIC, new ConnectionChecker());
+    }
+
+    @Override
+    protected void doStop() {
+        FutureUtils.cancel(backgroundFuture);
+    }
+
+    @Override
+    protected void doClose() {
+
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
index 78eef316332..c90edee0d50 100644
--- a/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
+++ b/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
@@ -69,15 +69,17 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
         private final State state;
         private final SnapshotId snapshotId;
         private final boolean includeGlobalState;
+        private final boolean partial;
         private final ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards;
         private final List<String> indices;
         private final ImmutableOpenMap<String, List<ShardId>> waitingIndices;
         private final long startTime;
 
-        public Entry(SnapshotId snapshotId, boolean includeGlobalState, State state, List<String> indices, long startTime, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
+        public Entry(SnapshotId snapshotId, boolean includeGlobalState, boolean partial, State state, List<String> indices, long startTime, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
             this.state = state;
             this.snapshotId = snapshotId;
             this.includeGlobalState = includeGlobalState;
+            this.partial = partial;
             this.indices = indices;
             this.startTime = startTime;
             if (shards == null) {
@@ -90,7 +92,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
         }
 
         public Entry(Entry entry, State state, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
-            this(entry.snapshotId, entry.includeGlobalState, state, entry.indices, entry.startTime, shards);
+            this(entry.snapshotId, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime, shards);
         }
 
         public Entry(Entry entry, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
@@ -121,6 +123,10 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
             return includeGlobalState;
         }
 
+        public boolean partial() {
+            return partial;
+        }
+
         public long startTime() {
             return startTime;
         }
@@ -133,6 +139,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
             Entry entry = (Entry) o;
 
             if (includeGlobalState != entry.includeGlobalState) return false;
+            if (partial != entry.partial) return false;
             if (startTime != entry.startTime) return false;
             if (!indices.equals(entry.indices)) return false;
             if (!shards.equals(entry.shards)) return false;
@@ -148,6 +155,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
             int result = state.hashCode();
             result = 31 * result + snapshotId.hashCode();
             result = 31 * result + (includeGlobalState ? 1 : 0);
+            result = 31 * result + (partial ? 1 : 0);
             result = 31 * result + shards.hashCode();
             result = 31 * result + indices.hashCode();
             result = 31 * result + waitingIndices.hashCode();
@@ -360,6 +368,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
         for (int i = 0; i < entries.length; i++) {
             SnapshotId snapshotId = SnapshotId.readSnapshotId(in);
             boolean includeGlobalState = in.readBoolean();
+            boolean partial = in.readBoolean();
             State state = State.fromValue(in.readByte());
             int indices = in.readVInt();
             List<String> indexBuilder = new ArrayList<>();
@@ -375,7 +384,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
                 State shardState = State.fromValue(in.readByte());
                 builder.put(shardId, new ShardSnapshotStatus(nodeId, shardState));
             }
-            entries[i] = new Entry(snapshotId, includeGlobalState, state, Collections.unmodifiableList(indexBuilder), startTime, builder.build());
+            entries[i] = new Entry(snapshotId, includeGlobalState, partial, state, Collections.unmodifiableList(indexBuilder), startTime, builder.build());
         }
         return new SnapshotsInProgress(entries);
     }
@@ -386,6 +395,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
         for (Entry entry : entries) {
             entry.snapshotId().writeTo(out);
             out.writeBoolean(entry.includeGlobalState());
+            out.writeBoolean(entry.partial());
             out.writeByte(entry.state().value());
             out.writeVInt(entry.indices().size());
             for (String index : entry.indices()) {
@@ -406,6 +416,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
         static final XContentBuilderString SNAPSHOTS = new XContentBuilderString("snapshots");
         static final XContentBuilderString SNAPSHOT = new XContentBuilderString("snapshot");
         static final XContentBuilderString INCLUDE_GLOBAL_STATE = new XContentBuilderString("include_global_state");
+        static final XContentBuilderString PARTIAL = new XContentBuilderString("partial");
         static final XContentBuilderString STATE = new XContentBuilderString("state");
         static final XContentBuilderString INDICES = new XContentBuilderString("indices");
         static final XContentBuilderString START_TIME_MILLIS = new XContentBuilderString("start_time_millis");
@@ -431,6 +442,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
         builder.field(Fields.REPOSITORY, entry.snapshotId().getRepository());
         builder.field(Fields.SNAPSHOT, entry.snapshotId().getSnapshot());
         builder.field(Fields.INCLUDE_GLOBAL_STATE, entry.includeGlobalState());
+        builder.field(Fields.PARTIAL, entry.partial());
         builder.field(Fields.STATE, entry.state());
         builder.startArray(Fields.INDICES);
         {
diff --git a/core/src/main/java/org/elasticsearch/cluster/ack/IndicesClusterStateUpdateRequest.java b/core/src/main/java/org/elasticsearch/cluster/ack/IndicesClusterStateUpdateRequest.java
index c691abe5906..33d716cb965 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ack/IndicesClusterStateUpdateRequest.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ack/IndicesClusterStateUpdateRequest.java
@@ -18,17 +18,19 @@
  */
 package org.elasticsearch.cluster.ack;
 
+import org.elasticsearch.index.Index;
+
 /**
  * Base cluster state update request that allows to execute update against multiple indices
  */
 public abstract class IndicesClusterStateUpdateRequest<T extends IndicesClusterStateUpdateRequest<T>> extends ClusterStateUpdateRequest<T> {
 
-    private String[] indices;
+    private Index[] indices;
 
     /**
      * Returns the indices the operation needs to be executed on
      */
-    public String[] indices() {
+    public Index[] indices() {
         return indices;
     }
 
@@ -36,7 +38,7 @@ public abstract class IndicesClusterStateUpdateRequest<T extends IndicesClusterS
      * Sets the indices the operation needs to be executed on
      */
     @SuppressWarnings("unchecked")
-    public T indices(String[] indices) {
+    public T indices(Index[] indices) {
         this.indices = indices;
         return (T)this;
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java b/core/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java
index 0e61712b010..12e5ea6d5d9 100644
--- a/core/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java
+++ b/core/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java
@@ -19,17 +19,17 @@
 
 package org.elasticsearch.cluster.action.index;
 
-import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.IndicesAdminClient;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.Mapping;
 
@@ -41,7 +41,9 @@ import java.util.concurrent.TimeoutException;
  */
 public class MappingUpdatedAction extends AbstractComponent {
 
-    public static final Setting<TimeValue> INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING = Setting.positiveTimeSetting("indices.mapping.dynamic_timeout", TimeValue.timeValueSeconds(30), true, Setting.Scope.CLUSTER);
+    public static final Setting<TimeValue> INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING =
+        Setting.positiveTimeSetting("indices.mapping.dynamic_timeout", TimeValue.timeValueSeconds(30),
+            Property.Dynamic, Property.NodeScope);
 
     private IndicesAdminClient client;
     private volatile TimeValue dynamicMappingUpdateTimeout;
@@ -62,48 +64,20 @@ public class MappingUpdatedAction extends AbstractComponent {
         this.client = client.admin().indices();
     }
 
-    private PutMappingRequestBuilder updateMappingRequest(String index, String type, Mapping mappingUpdate, final TimeValue timeout) {
+    private PutMappingRequestBuilder updateMappingRequest(Index index, String type, Mapping mappingUpdate, final TimeValue timeout) {
         if (type.equals(MapperService.DEFAULT_MAPPING)) {
             throw new IllegalArgumentException("_default_ mapping should not be updated");
         }
-        return client.preparePutMapping(index).setType(type).setSource(mappingUpdate.toString())
+        return client.preparePutMapping().setConcreteIndex(index).setType(type).setSource(mappingUpdate.toString())
                 .setMasterNodeTimeout(timeout).setTimeout(timeout);
     }
 
-    public void updateMappingOnMaster(String index, String type, Mapping mappingUpdate, final TimeValue timeout, final MappingUpdateListener listener) {
-        final PutMappingRequestBuilder request = updateMappingRequest(index, type, mappingUpdate, timeout);
-        if (listener == null) {
-            request.execute();
-        } else {
-            final ActionListener<PutMappingResponse> actionListener = new ActionListener<PutMappingResponse>() {
-                @Override
-                public void onResponse(PutMappingResponse response) {
-                    if (response.isAcknowledged()) {
-                        listener.onMappingUpdate();
-                    } else {
-                        listener.onFailure(new TimeoutException("Failed to acknowledge the mapping response within [" + timeout + "]"));
-                    }
-                }
-
-                @Override
-                public void onFailure(Throwable e) {
-                    listener.onFailure(e);
-                }
-            };
-            request.execute(actionListener);
-        }
-    }
-
-    public void updateMappingOnMasterAsynchronously(String index, String type, Mapping mappingUpdate) throws Exception {
-        updateMappingOnMaster(index, type, mappingUpdate, dynamicMappingUpdateTimeout, null);
-    }
-
     /**
-     * Same as {@link #updateMappingOnMasterSynchronously(String, String, Mapping, TimeValue)}
+     * Same as {@link #updateMappingOnMaster(Index, String, Mapping, TimeValue)}
      * using the default timeout.
      */
-    public void updateMappingOnMasterSynchronously(String index, String type, Mapping mappingUpdate) throws Exception {
-        updateMappingOnMasterSynchronously(index, type, mappingUpdate, dynamicMappingUpdateTimeout);
+    public void updateMappingOnMaster(Index index, String type, Mapping mappingUpdate) throws Exception {
+        updateMappingOnMaster(index, type, mappingUpdate, dynamicMappingUpdateTimeout);
     }
 
     /**
@@ -111,19 +85,9 @@ public class MappingUpdatedAction extends AbstractComponent {
      * {@code timeout}. When this method returns successfully mappings have
      * been applied to the master node and propagated to data nodes.
      */
-    public void updateMappingOnMasterSynchronously(String index, String type, Mapping mappingUpdate, TimeValue timeout) throws Exception {
+    public void updateMappingOnMaster(Index index, String type, Mapping mappingUpdate, TimeValue timeout) throws Exception {
         if (updateMappingRequest(index, type, mappingUpdate, timeout).get().isAcknowledged() == false) {
             throw new TimeoutException("Failed to acknowledge mapping update within [" + timeout + "]");
         }
     }
-
-    /**
-     * A listener to be notified when the mappings were updated
-     */
-    public static interface MappingUpdateListener {
-
-        void onMappingUpdate();
-
-        void onFailure(Throwable t);
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/action/index/NodeIndexDeletedAction.java b/core/src/main/java/org/elasticsearch/cluster/action/index/NodeIndexDeletedAction.java
index 012cc66e110..93fce95fc23 100644
--- a/core/src/main/java/org/elasticsearch/cluster/action/index/NodeIndexDeletedAction.java
+++ b/core/src/main/java/org/elasticsearch/cluster/action/index/NodeIndexDeletedAction.java
@@ -76,18 +76,18 @@ public class NodeIndexDeletedAction extends AbstractComponent {
         listeners.remove(listener);
     }
 
-    public void nodeIndexDeleted(final ClusterState clusterState, final String index, final IndexSettings indexSettings, final String nodeId) {
+    public void nodeIndexDeleted(final ClusterState clusterState, final Index index, final IndexSettings indexSettings, final String nodeId) {
         final DiscoveryNodes nodes = clusterState.nodes();
         transportService.sendRequest(clusterState.nodes().masterNode(),
                 INDEX_DELETED_ACTION_NAME, new NodeIndexDeletedMessage(index, nodeId), EmptyTransportResponseHandler.INSTANCE_SAME);
         if (nodes.localNode().isDataNode() == false) {
-            logger.trace("[{}] not acking store deletion (not a data node)");
+            logger.trace("[{}] not acking store deletion (not a data node)", index);
             return;
         }
         threadPool.generic().execute(new AbstractRunnable() {
             @Override
             public void onFailure(Throwable t) {
-                logger.warn("[{}]failed to ack index store deleted for  index", t, index);
+                logger.warn("[{}] failed to ack index store deleted for index", t, index);
             }
 
             @Override
@@ -97,7 +97,7 @@ public class NodeIndexDeletedAction extends AbstractComponent {
         });
     }
 
-    private void lockIndexAndAck(String index, DiscoveryNodes nodes, String nodeId, ClusterState clusterState, IndexSettings indexSettings) throws IOException {
+    private void lockIndexAndAck(Index index, DiscoveryNodes nodes, String nodeId, ClusterState clusterState, IndexSettings indexSettings) throws IOException {
         try {
             // we are waiting until we can lock the index / all shards on the node and then we ack the delete of the store to the
             // master. If we can't acquire the locks here immediately there might be a shard of this index still holding on to the lock
@@ -114,9 +114,9 @@ public class NodeIndexDeletedAction extends AbstractComponent {
     }
 
     public interface Listener {
-        void onNodeIndexDeleted(String index, String nodeId);
+        void onNodeIndexDeleted(Index index, String nodeId);
 
-        void onNodeIndexStoreDeleted(String index, String nodeId);
+        void onNodeIndexStoreDeleted(Index index, String nodeId);
     }
 
     private class NodeIndexDeletedTransportHandler implements TransportRequestHandler<NodeIndexDeletedMessage> {
@@ -143,13 +143,13 @@ public class NodeIndexDeletedAction extends AbstractComponent {
 
     public static class NodeIndexDeletedMessage extends TransportRequest {
 
-        String index;
+        Index index;
         String nodeId;
 
         public NodeIndexDeletedMessage() {
         }
 
-        NodeIndexDeletedMessage(String index, String nodeId) {
+        NodeIndexDeletedMessage(Index index, String nodeId) {
             this.index = index;
             this.nodeId = nodeId;
         }
@@ -157,27 +157,27 @@ public class NodeIndexDeletedAction extends AbstractComponent {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            out.writeString(index);
+            index.writeTo(out);
             out.writeString(nodeId);
         }
 
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
-            index = in.readString();
+            index = new Index(in);
             nodeId = in.readString();
         }
     }
 
     public static class NodeIndexStoreDeletedMessage extends TransportRequest {
 
-        String index;
+        Index index;
         String nodeId;
 
         public NodeIndexStoreDeletedMessage() {
         }
 
-        NodeIndexStoreDeletedMessage(String index, String nodeId) {
+        NodeIndexStoreDeletedMessage(Index index, String nodeId) {
             this.index = index;
             this.nodeId = nodeId;
         }
@@ -185,14 +185,14 @@ public class NodeIndexDeletedAction extends AbstractComponent {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            out.writeString(index);
+            index.writeTo(out);
             out.writeString(nodeId);
         }
 
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
-            index = in.readString();
+            index = new Index(in);
             nodeId = in.readString();
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
index f0cd8cd9ca9..5d7dd0b830c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
+++ b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
@@ -21,7 +21,6 @@ package org.elasticsearch.cluster.action.shard;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.ClusterStateTaskConfig;
@@ -37,6 +36,7 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractComponent;
@@ -151,7 +151,7 @@ public class ShardStateAction extends AbstractComponent {
             @Override
             public void onNewClusterState(ClusterState state) {
                 if (logger.isTraceEnabled()) {
-                    logger.trace("new cluster state [{}] after waiting for master election to fail shard [{}]", shardRoutingEntry.getShardRouting().shardId(), state.prettyPrint(), shardRoutingEntry);
+                    logger.trace("new cluster state [{}] after waiting for master election to fail shard [{}]", state.prettyPrint(), shardRoutingEntry);
                 }
                 sendShardAction(actionName, observer, shardRoutingEntry, listener);
             }
@@ -321,7 +321,7 @@ public class ShardStateAction extends AbstractComponent {
             if (numberOfUnassignedShards > 0) {
                 String reason = String.format(Locale.ROOT, "[%d] unassigned shards after failing shards", numberOfUnassignedShards);
                 if (logger.isTraceEnabled()) {
-                    logger.trace(reason + ", scheduling a reroute");
+                    logger.trace("{}, scheduling a reroute", reason);
                 }
                 routingService.reroute(reason);
             }
diff --git a/core/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java b/core/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java
index d66a2437ef2..42ab496fe33 100644
--- a/core/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java
+++ b/core/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java
@@ -79,7 +79,7 @@ public final class ClusterStateHealth implements Iterable<ClusterIndexHealth>, S
      * @param clusterState The current cluster state. Must not be null.
      */
     public ClusterStateHealth(ClusterState clusterState) {
-        this(clusterState, clusterState.metaData().concreteAllIndices());
+        this(clusterState, clusterState.metaData().getConcreteAllIndices());
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java b/core/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java
index d9b288bb897..4b4a8e54d7c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java
@@ -20,6 +20,7 @@ package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 
 /**
  * This class acts as a functional wrapper around the <tt>index.auto_expand_replicas</tt> setting.
@@ -56,7 +57,7 @@ final class AutoExpandReplicas {
             }
         }
         return new AutoExpandReplicas(min, max, true);
-    }, true, Setting.Scope.INDEX);
+    }, Property.Dynamic, Property.IndexScope);
 
     private final int minReplicas;
     private final int maxReplicas;
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
index 7b4d5a68cec..ca3c153e1d6 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import com.carrotsearch.hppc.LongArrayList;
 import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
@@ -29,6 +30,8 @@ import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.node.DiscoveryNodeFilters;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.collect.ImmutableOpenIntMap;
@@ -38,6 +41,7 @@ import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.loader.SettingsLoader;
 import org.elasticsearch.common.xcontent.FromXContentBuilder;
@@ -45,6 +49,8 @@ import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.gateway.MetaDataStateFormat;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.rest.RestStatus;
@@ -53,6 +59,7 @@ import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.text.ParseException;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -152,28 +159,36 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
 
     public static final String INDEX_SETTING_PREFIX = "index.";
     public static final String SETTING_NUMBER_OF_SHARDS = "index.number_of_shards";
-    public static final Setting<Integer> INDEX_NUMBER_OF_SHARDS_SETTING = Setting.intSetting(SETTING_NUMBER_OF_SHARDS, 5, 1, false, Setting.Scope.INDEX);
+    public static final Setting<Integer> INDEX_NUMBER_OF_SHARDS_SETTING =
+        Setting.intSetting(SETTING_NUMBER_OF_SHARDS, 5, 1, Property.IndexScope);
     public static final String SETTING_NUMBER_OF_REPLICAS = "index.number_of_replicas";
-    public static final Setting<Integer> INDEX_NUMBER_OF_REPLICAS_SETTING = Setting.intSetting(SETTING_NUMBER_OF_REPLICAS, 1, 0, true, Setting.Scope.INDEX);
+    public static final Setting<Integer> INDEX_NUMBER_OF_REPLICAS_SETTING =
+        Setting.intSetting(SETTING_NUMBER_OF_REPLICAS, 1, 0, Property.Dynamic, Property.IndexScope);
     public static final String SETTING_SHADOW_REPLICAS = "index.shadow_replicas";
-    public static final Setting<Boolean> INDEX_SHADOW_REPLICAS_SETTING = Setting.boolSetting(SETTING_SHADOW_REPLICAS, false, false, Setting.Scope.INDEX);
+    public static final Setting<Boolean> INDEX_SHADOW_REPLICAS_SETTING =
+        Setting.boolSetting(SETTING_SHADOW_REPLICAS, false, Property.IndexScope);
 
     public static final String SETTING_SHARED_FILESYSTEM = "index.shared_filesystem";
-    public static final Setting<Boolean> INDEX_SHARED_FILESYSTEM_SETTING = Setting.boolSetting(SETTING_SHARED_FILESYSTEM, false, false, Setting.Scope.INDEX);
+    public static final Setting<Boolean> INDEX_SHARED_FILESYSTEM_SETTING =
+        Setting.boolSetting(SETTING_SHARED_FILESYSTEM, false, Property.IndexScope);
 
     public static final String SETTING_AUTO_EXPAND_REPLICAS = "index.auto_expand_replicas";
     public static final Setting<AutoExpandReplicas> INDEX_AUTO_EXPAND_REPLICAS_SETTING = AutoExpandReplicas.SETTING;
     public static final String SETTING_READ_ONLY = "index.blocks.read_only";
-    public static final Setting<Boolean> INDEX_READ_ONLY_SETTING = Setting.boolSetting(SETTING_READ_ONLY, false, true, Setting.Scope.INDEX);
+    public static final Setting<Boolean> INDEX_READ_ONLY_SETTING =
+        Setting.boolSetting(SETTING_READ_ONLY, false, Property.Dynamic, Property.IndexScope);
 
     public static final String SETTING_BLOCKS_READ = "index.blocks.read";
-    public static final Setting<Boolean> INDEX_BLOCKS_READ_SETTING = Setting.boolSetting(SETTING_BLOCKS_READ, false, true, Setting.Scope.INDEX);
+    public static final Setting<Boolean> INDEX_BLOCKS_READ_SETTING =
+        Setting.boolSetting(SETTING_BLOCKS_READ, false, Property.Dynamic, Property.IndexScope);
 
     public static final String SETTING_BLOCKS_WRITE = "index.blocks.write";
-    public static final Setting<Boolean> INDEX_BLOCKS_WRITE_SETTING = Setting.boolSetting(SETTING_BLOCKS_WRITE, false, true, Setting.Scope.INDEX);
+    public static final Setting<Boolean> INDEX_BLOCKS_WRITE_SETTING =
+        Setting.boolSetting(SETTING_BLOCKS_WRITE, false, Property.Dynamic, Property.IndexScope);
 
     public static final String SETTING_BLOCKS_METADATA = "index.blocks.metadata";
-    public static final Setting<Boolean> INDEX_BLOCKS_METADATA_SETTING = Setting.boolSetting(SETTING_BLOCKS_METADATA, false, true, Setting.Scope.INDEX);
+    public static final Setting<Boolean> INDEX_BLOCKS_METADATA_SETTING =
+        Setting.boolSetting(SETTING_BLOCKS_METADATA, false, Property.Dynamic, Property.IndexScope);
 
     public static final String SETTING_VERSION_CREATED = "index.version.created";
     public static final String SETTING_VERSION_CREATED_STRING = "index.version.created_string";
@@ -182,30 +197,45 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
     public static final String SETTING_VERSION_MINIMUM_COMPATIBLE = "index.version.minimum_compatible";
     public static final String SETTING_CREATION_DATE = "index.creation_date";
     public static final String SETTING_PRIORITY = "index.priority";
-    public static final Setting<Integer> INDEX_PRIORITY_SETTING = Setting.intSetting("index.priority", 1, 0, true, Setting.Scope.INDEX);
+    public static final Setting<Integer> INDEX_PRIORITY_SETTING =
+        Setting.intSetting("index.priority", 1, 0, Property.Dynamic, Property.IndexScope);
     public static final String SETTING_CREATION_DATE_STRING = "index.creation_date_string";
     public static final String SETTING_INDEX_UUID = "index.uuid";
     public static final String SETTING_DATA_PATH = "index.data_path";
-    public static final Setting<String> INDEX_DATA_PATH_SETTING = new Setting<>(SETTING_DATA_PATH, "", Function.identity(), false, Setting.Scope.INDEX);
+    public static final Setting<String> INDEX_DATA_PATH_SETTING =
+        new Setting<>(SETTING_DATA_PATH, "", Function.identity(), Property.IndexScope);
     public static final String SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE = "index.shared_filesystem.recover_on_any_node";
-    public static final Setting<Boolean> INDEX_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE_SETTING = Setting.boolSetting(SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE, false, true, Setting.Scope.INDEX);
+    public static final Setting<Boolean> INDEX_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE_SETTING =
+        Setting.boolSetting(SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE, false, Property.Dynamic, Property.IndexScope);
     public static final String INDEX_UUID_NA_VALUE = "_na_";
 
-    public static final Setting<Settings> INDEX_ROUTING_REQUIRE_GROUP_SETTING = Setting.groupSetting("index.routing.allocation.require.", true, Setting.Scope.INDEX);
-    public static final Setting<Settings> INDEX_ROUTING_INCLUDE_GROUP_SETTING = Setting.groupSetting("index.routing.allocation.include.", true, Setting.Scope.INDEX);
-    public static final Setting<Settings> INDEX_ROUTING_EXCLUDE_GROUP_SETTING = Setting.groupSetting("index.routing.allocation.exclude.", true, Setting.Scope.INDEX);
+    public static final Setting<Settings> INDEX_ROUTING_REQUIRE_GROUP_SETTING =
+        Setting.groupSetting("index.routing.allocation.require.", Property.Dynamic, Property.IndexScope);
+    public static final Setting<Settings> INDEX_ROUTING_INCLUDE_GROUP_SETTING =
+        Setting.groupSetting("index.routing.allocation.include.", Property.Dynamic, Property.IndexScope);
+    public static final Setting<Settings> INDEX_ROUTING_EXCLUDE_GROUP_SETTING =
+        Setting.groupSetting("index.routing.allocation.exclude.", Property.Dynamic, Property.IndexScope);
 
     public static final IndexMetaData PROTO = IndexMetaData.builder("")
             .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
             .numberOfShards(1).numberOfReplicas(0).build();
 
     public static final String KEY_ACTIVE_ALLOCATIONS = "active_allocations";
+    static final String KEY_VERSION = "version";
+    static final String KEY_SETTINGS = "settings";
+    static final String KEY_STATE = "state";
+    static final String KEY_MAPPINGS = "mappings";
+    static final String KEY_ALIASES = "aliases";
+    public static final String KEY_PRIMARY_TERMS = "primary_terms";
+
+    public static final String INDEX_STATE_FILE_PREFIX = "state-";
 
     private final int numberOfShards;
     private final int numberOfReplicas;
 
     private final Index index;
     private final long version;
+    private final long[] primaryTerms;
 
     private final State state;
 
@@ -229,7 +259,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
     private final Version indexUpgradedVersion;
     private final org.apache.lucene.util.Version minimumCompatibleLuceneVersion;
 
-    private IndexMetaData(Index index, long version, State state, int numberOfShards, int numberOfReplicas, Settings settings,
+    private IndexMetaData(Index index, long version, long[] primaryTerms, State state, int numberOfShards, int numberOfReplicas, Settings settings,
                           ImmutableOpenMap<String, MappingMetaData> mappings, ImmutableOpenMap<String, AliasMetaData> aliases,
                           ImmutableOpenMap<String, Custom> customs, ImmutableOpenIntMap<Set<String>> activeAllocationIds,
                           DiscoveryNodeFilters requireFilters, DiscoveryNodeFilters includeFilters, DiscoveryNodeFilters excludeFilters,
@@ -237,6 +267,8 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
 
         this.index = index;
         this.version = version;
+        this.primaryTerms = primaryTerms;
+        assert primaryTerms.length == numberOfShards;
         this.state = state;
         this.numberOfShards = numberOfShards;
         this.numberOfReplicas = numberOfReplicas;
@@ -278,6 +310,16 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         return this.version;
     }
 
+
+    /**
+     * The term of the current selected primary. This is a non-negative number incremented when
+     * a primary shard is assigned after a full cluster restart or a replica shard is promoted to a primary
+     * See {@link AllocationService#updateMetaDataWithRoutingTable(MetaData, RoutingTable, RoutingTable)}.
+     **/
+    public long primaryTerm(int shardId) {
+        return this.primaryTerms[shardId];
+    }
+
     /**
      * Return the {@link Version} on which this index has been created. This
      * information is typically useful for backward compatibility.
@@ -398,6 +440,10 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
 
         IndexMetaData that = (IndexMetaData) o;
 
+        if (version != that.version) {
+            return false;
+        }
+
         if (!aliases.equals(that.aliases)) {
             return false;
         }
@@ -416,6 +462,10 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         if (!customs.equals(that.customs)) {
             return false;
         }
+
+        if (Arrays.equals(primaryTerms, that.primaryTerms) == false) {
+            return false;
+        }
         if (!activeAllocationIds.equals(that.activeAllocationIds)) {
             return false;
         }
@@ -425,14 +475,18 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
     @Override
     public int hashCode() {
         int result = index.hashCode();
+        result = 31 * result + Long.hashCode(version);
         result = 31 * result + state.hashCode();
         result = 31 * result + aliases.hashCode();
         result = 31 * result + settings.hashCode();
         result = 31 * result + mappings.hashCode();
+        result = 31 * result + customs.hashCode();
+        result = 31 * result + Arrays.hashCode(primaryTerms);
         result = 31 * result + activeAllocationIds.hashCode();
         return result;
     }
 
+
     @Override
     public Diff<IndexMetaData> diff(IndexMetaData previousState) {
         return new IndexMetaDataDiff(previousState, this);
@@ -458,6 +512,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
 
         private final String index;
         private final long version;
+        private final long[] primaryTerms;
         private final State state;
         private final Settings settings;
         private final Diff<ImmutableOpenMap<String, MappingMetaData>> mappings;
@@ -470,11 +525,12 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
             version = after.version;
             state = after.state;
             settings = after.settings;
+            primaryTerms = after.primaryTerms;
             mappings = DiffableUtils.diff(before.mappings, after.mappings, DiffableUtils.getStringKeySerializer());
             aliases = DiffableUtils.diff(before.aliases, after.aliases, DiffableUtils.getStringKeySerializer());
             customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer());
             activeAllocationIds = DiffableUtils.diff(before.activeAllocationIds, after.activeAllocationIds,
-                    DiffableUtils.getVIntKeySerializer(), DiffableUtils.StringSetValueSerializer.getInstance());
+                DiffableUtils.getVIntKeySerializer(), DiffableUtils.StringSetValueSerializer.getInstance());
         }
 
         public IndexMetaDataDiff(StreamInput in) throws IOException {
@@ -482,22 +538,23 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
             version = in.readLong();
             state = State.fromId(in.readByte());
             settings = Settings.readSettingsFromStream(in);
+            primaryTerms = in.readVLongArray();
             mappings = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), MappingMetaData.PROTO);
             aliases = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), AliasMetaData.PROTO);
             customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(),
-                    new DiffableUtils.DiffableValueSerializer<String, Custom>() {
-                        @Override
-                        public Custom read(StreamInput in, String key) throws IOException {
-                            return lookupPrototypeSafe(key).readFrom(in);
-                        }
+                new DiffableUtils.DiffableValueSerializer<String, Custom>() {
+                    @Override
+                    public Custom read(StreamInput in, String key) throws IOException {
+                        return lookupPrototypeSafe(key).readFrom(in);
+                    }
 
-                        @Override
-                        public Diff<Custom> readDiff(StreamInput in, String key) throws IOException {
-                            return lookupPrototypeSafe(key).readDiffFrom(in);
-                        }
-                    });
+                    @Override
+                    public Diff<Custom> readDiff(StreamInput in, String key) throws IOException {
+                        return lookupPrototypeSafe(key).readDiffFrom(in);
+                    }
+                });
             activeAllocationIds = DiffableUtils.readImmutableOpenIntMapDiff(in, DiffableUtils.getVIntKeySerializer(),
-                    DiffableUtils.StringSetValueSerializer.getInstance());
+                DiffableUtils.StringSetValueSerializer.getInstance());
         }
 
         @Override
@@ -506,6 +563,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
             out.writeLong(version);
             out.writeByte(state.id);
             Settings.writeSettingsToStream(settings, out);
+            out.writeVLongArray(primaryTerms);
             mappings.writeTo(out);
             aliases.writeTo(out);
             customs.writeTo(out);
@@ -518,6 +576,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
             builder.version(version);
             builder.state(state);
             builder.settings(settings);
+            builder.primaryTerms(primaryTerms);
             builder.mappings.putAll(mappings.apply(part.mappings));
             builder.aliases.putAll(aliases.apply(part.aliases));
             builder.customs.putAll(customs.apply(part.customs));
@@ -532,6 +591,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         builder.version(in.readLong());
         builder.state(State.fromId(in.readByte()));
         builder.settings(readSettingsFromStream(in));
+        builder.primaryTerms(in.readVLongArray());
         int mappingsSize = in.readVInt();
         for (int i = 0; i < mappingsSize; i++) {
             MappingMetaData mappingMd = MappingMetaData.PROTO.readFrom(in);
@@ -563,6 +623,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         out.writeLong(version);
         out.writeByte(state.id());
         writeSettingsToStream(settings, out);
+        out.writeVLongArray(primaryTerms);
         out.writeVInt(mappings.size());
         for (ObjectCursor<MappingMetaData> cursor : mappings.values()) {
             cursor.value.writeTo(out);
@@ -596,6 +657,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         private String index;
         private State state = State.OPEN;
         private long version = 1;
+        private long[] primaryTerms = null;
         private Settings settings = Settings.Builder.EMPTY_SETTINGS;
         private final ImmutableOpenMap.Builder<String, MappingMetaData> mappings;
         private final ImmutableOpenMap.Builder<String, AliasMetaData> aliases;
@@ -615,6 +677,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
             this.state = indexMetaData.state;
             this.version = indexMetaData.version;
             this.settings = indexMetaData.getSettings();
+            this.primaryTerms = indexMetaData.primaryTerms.clone();
             this.mappings = ImmutableOpenMap.builder(indexMetaData.mappings);
             this.aliases = ImmutableOpenMap.builder(indexMetaData.aliases);
             this.customs = ImmutableOpenMap.builder(indexMetaData.customs);
@@ -654,8 +717,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         }
 
         public Builder settings(Settings.Builder settings) {
-            this.settings = settings.build();
-            return this;
+            return settings(settings.build());
         }
 
         public Builder settings(Settings settings) {
@@ -723,6 +785,42 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
             return this;
         }
 
+        /**
+         * returns the primary term for the given shard.
+         * See {@link IndexMetaData#primaryTerm(int)} for more information.
+         */
+        public long primaryTerm(int shardId) {
+            if (primaryTerms == null) {
+                initializePrimaryTerms();
+            }
+            return this.primaryTerms[shardId];
+        }
+
+        /**
+         * sets the primary term for the given shard.
+         * See {@link IndexMetaData#primaryTerm(int)} for more information.
+         */
+        public Builder primaryTerm(int shardId, long primaryTerm) {
+            if (primaryTerms == null) {
+                initializePrimaryTerms();
+            }
+            this.primaryTerms[shardId] = primaryTerm;
+            return this;
+        }
+
+        private void primaryTerms(long[] primaryTerms) {
+            this.primaryTerms = primaryTerms.clone();
+        }
+
+        private void initializePrimaryTerms() {
+            assert primaryTerms == null;
+            if (numberOfShards() < 0) {
+                throw new IllegalStateException("you must set the number of shards before setting/reading primary terms");
+            }
+            primaryTerms = new long[numberOfShards()];
+        }
+
+
         public IndexMetaData build() {
             ImmutableOpenMap.Builder<String, AliasMetaData> tmpAliases = aliases;
             Settings tmpSettings = settings;
@@ -797,27 +895,34 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
                 minimumCompatibleLuceneVersion = null;
             }
 
+            if (primaryTerms == null) {
+                initializePrimaryTerms();
+            } else if (primaryTerms.length != numberOfShards) {
+                throw new IllegalStateException("primaryTerms length is [" + primaryTerms.length
+                    + "] but should be equal to number of shards [" + numberOfShards() + "]");
+            }
+
             final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE);
-            return new IndexMetaData(new Index(index, uuid), version, state, numberOfShards, numberOfReplicas, tmpSettings, mappings.build(),
-                    tmpAliases.build(), customs.build(), filledActiveAllocationIds.build(), requireFilters, includeFilters, excludeFilters,
-                    indexCreatedVersion, indexUpgradedVersion, minimumCompatibleLuceneVersion);
+            return new IndexMetaData(new Index(index, uuid), version, primaryTerms, state, numberOfShards, numberOfReplicas, tmpSettings, mappings.build(),
+                tmpAliases.build(), customs.build(), filledActiveAllocationIds.build(), requireFilters, includeFilters, excludeFilters,
+                indexCreatedVersion, indexUpgradedVersion, minimumCompatibleLuceneVersion);
         }
 
         public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
             builder.startObject(indexMetaData.getIndex().getName(), XContentBuilder.FieldCaseConversion.NONE);
 
-            builder.field("version", indexMetaData.getVersion());
-            builder.field("state", indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH));
+            builder.field(KEY_VERSION, indexMetaData.getVersion());
+            builder.field(KEY_STATE, indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH));
 
             boolean binary = params.paramAsBoolean("binary", false);
 
-            builder.startObject("settings");
+            builder.startObject(KEY_SETTINGS);
             for (Map.Entry<String, String> entry : indexMetaData.getSettings().getAsMap().entrySet()) {
                 builder.field(entry.getKey(), entry.getValue());
             }
             builder.endObject();
 
-            builder.startArray("mappings");
+            builder.startArray(KEY_MAPPINGS);
             for (ObjectObjectCursor<String, MappingMetaData> cursor : indexMetaData.getMappings()) {
                 if (binary) {
                     builder.value(cursor.value.source().compressed());
@@ -837,12 +942,18 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
                 builder.endObject();
             }
 
-            builder.startObject("aliases");
+            builder.startObject(KEY_ALIASES);
             for (ObjectCursor<AliasMetaData> cursor : indexMetaData.getAliases().values()) {
                 AliasMetaData.Builder.toXContent(cursor.value, builder, params);
             }
             builder.endObject();
 
+            builder.startArray(KEY_PRIMARY_TERMS);
+            for (int i = 0; i < indexMetaData.getNumberOfShards(); i++) {
+                builder.value(indexMetaData.primaryTerm(i));
+            }
+            builder.endArray();
+
             builder.startObject(KEY_ACTIVE_ALLOCATIONS);
             for (IntObjectCursor<Set<String>> cursor : indexMetaData.activeAllocationIds) {
                 builder.startArray(String.valueOf(cursor.key));
@@ -877,9 +988,9 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
                 if (token == XContentParser.Token.FIELD_NAME) {
                     currentFieldName = parser.currentName();
                 } else if (token == XContentParser.Token.START_OBJECT) {
-                    if ("settings".equals(currentFieldName)) {
+                    if (KEY_SETTINGS.equals(currentFieldName)) {
                         builder.settings(Settings.settingsBuilder().put(SettingsLoader.Helper.loadNestedFromMap(parser.mapOrdered())));
-                    } else if ("mappings".equals(currentFieldName)) {
+                    } else if (KEY_MAPPINGS.equals(currentFieldName)) {
                         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                             if (token == XContentParser.Token.FIELD_NAME) {
                                 currentFieldName = parser.currentName();
@@ -891,7 +1002,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
                                 throw new IllegalArgumentException("Unexpected token: " + token);
                             }
                         }
-                    } else if ("aliases".equals(currentFieldName)) {
+                    } else if (KEY_ALIASES.equals(currentFieldName)) {
                         while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
                             builder.putAlias(AliasMetaData.Builder.fromXContent(parser));
                         }
@@ -931,7 +1042,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
                         }
                     }
                 } else if (token == XContentParser.Token.START_ARRAY) {
-                    if ("mappings".equals(currentFieldName)) {
+                    if (KEY_MAPPINGS.equals(currentFieldName)) {
                         while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                             if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
                                 builder.putMapping(new MappingMetaData(new CompressedXContent(parser.binaryValue())));
@@ -943,13 +1054,23 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
                                 }
                             }
                         }
+                    } else if (KEY_PRIMARY_TERMS.equals(currentFieldName)) {
+                        LongArrayList list = new LongArrayList();
+                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                            if (token == XContentParser.Token.VALUE_NUMBER) {
+                                list.add(parser.longValue());
+                            } else {
+                                throw new IllegalStateException("found a non-numeric value under [" + KEY_PRIMARY_TERMS + "]");
+                            }
+                        }
+                        builder.primaryTerms(list.toArray());
                     } else {
                         throw new IllegalArgumentException("Unexpected field for an array " + currentFieldName);
                     }
                 } else if (token.isValue()) {
-                    if ("state".equals(currentFieldName)) {
+                    if (KEY_STATE.equals(currentFieldName)) {
                         builder.state(State.fromString(parser.text()));
-                    } else if ("version".equals(currentFieldName)) {
+                    } else if (KEY_VERSION.equals(currentFieldName)) {
                         builder.version(parser.longValue());
                     } else {
                         throw new IllegalArgumentException("Unexpected field [" + currentFieldName + "]");
@@ -1008,4 +1129,21 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         return builder.build();
     }
 
+    private static final ToXContent.Params FORMAT_PARAMS = new MapParams(Collections.singletonMap("binary", "true"));
+
+    /**
+     * State format for {@link IndexMetaData} to write to and load from disk
+     */
+    public static final MetaDataStateFormat<IndexMetaData> FORMAT = new MetaDataStateFormat<IndexMetaData>(XContentType.SMILE, INDEX_STATE_FILE_PREFIX) {
+
+        @Override
+        public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOException {
+            Builder.toXContent(state, builder, FORMAT_PARAMS);
+        }
+
+        @Override
+        public IndexMetaData fromXContent(XContentParser parser) throws IOException {
+            return Builder.fromXContent(parser);
+        }
+    };
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index cca633a7651..2abbea04d51 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -32,6 +32,7 @@ import org.elasticsearch.common.joda.DateMathParser;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.indices.IndexClosedException;
 import org.joda.time.DateTimeZone;
@@ -65,11 +66,20 @@ public class IndexNameExpressionResolver extends AbstractComponent {
         );
     }
 
+    /**
+     * Same as {@link #concreteIndexNames(ClusterState, IndicesOptions, String...)}, but the index expressions and options
+     * are encapsulated in the specified request.
+     */
+    public String[] concreteIndexNames(ClusterState state, IndicesRequest request) {
+        Context context = new Context(state, request.indicesOptions());
+        return concreteIndexNames(context, request.indices());
+    }
+
     /**
      * Same as {@link #concreteIndices(ClusterState, IndicesOptions, String...)}, but the index expressions and options
      * are encapsulated in the specified request.
      */
-    public String[] concreteIndices(ClusterState state, IndicesRequest request) {
+    public Index[] concreteIndices(ClusterState state, IndicesRequest request) {
         Context context = new Context(state, request.indicesOptions());
         return concreteIndices(context, request.indices());
     }
@@ -87,7 +97,25 @@ public class IndexNameExpressionResolver extends AbstractComponent {
      * @throws IllegalArgumentException if one of the aliases resolve to multiple indices and the provided
      * indices options in the context don't allow such a case.
      */
-    public String[] concreteIndices(ClusterState state, IndicesOptions options, String... indexExpressions) {
+    public String[] concreteIndexNames(ClusterState state, IndicesOptions options, String... indexExpressions) {
+        Context context = new Context(state, options);
+        return concreteIndexNames(context, indexExpressions);
+    }
+
+    /**
+     * Translates the provided index expression into actual concrete indices, properly deduplicated.
+     *
+     * @param state             the cluster state containing all the data to resolve to expressions to concrete indices
+     * @param options           defines how the aliases or indices need to be resolved to concrete indices
+     * @param indexExpressions  expressions that can be resolved to alias or index names.
+     * @return the resolved concrete indices based on the cluster state, indices options and index expressions
+     * @throws IndexNotFoundException if one of the index expressions is pointing to a missing index or alias and the
+     * provided indices options in the context don't allow such a case, or if the final result of the indices resolution
+     * contains no indices and the indices options in the context don't allow such a case.
+     * @throws IllegalArgumentException if one of the aliases resolve to multiple indices and the provided
+     * indices options in the context don't allow such a case.
+     */
+    public Index[] concreteIndices(ClusterState state, IndicesOptions options, String... indexExpressions) {
         Context context = new Context(state, options);
         return concreteIndices(context, indexExpressions);
     }
@@ -105,12 +133,21 @@ public class IndexNameExpressionResolver extends AbstractComponent {
      * @throws IllegalArgumentException if one of the aliases resolve to multiple indices and the provided
      * indices options in the context don't allow such a case.
      */
-    public String[] concreteIndices(ClusterState state, IndicesOptions options, long startTime, String... indexExpressions) {
+    public String[] concreteIndexNames(ClusterState state, IndicesOptions options, long startTime, String... indexExpressions) {
         Context context = new Context(state, options, startTime);
-        return concreteIndices(context, indexExpressions);
+        return concreteIndexNames(context, indexExpressions);
     }
 
-    String[] concreteIndices(Context context, String... indexExpressions) {
+    String[] concreteIndexNames(Context context, String... indexExpressions) {
+        Index[] indexes = concreteIndices(context, indexExpressions);
+        String[] names = new String[indexes.length];
+        for (int i = 0; i < indexes.length; i++) {
+            names[i] = indexes[i].getName();
+        }
+        return names;
+    }
+
+    Index[] concreteIndices(Context context, String... indexExpressions) {
         if (indexExpressions == null || indexExpressions.length == 0) {
             indexExpressions = new String[]{MetaData.ALL};
         }
@@ -136,11 +173,11 @@ public class IndexNameExpressionResolver extends AbstractComponent {
                 infe.setResources("index_expression", indexExpressions);
                 throw infe;
             } else {
-                return Strings.EMPTY_ARRAY;
+                return Index.EMPTY_ARRAY;
             }
         }
 
-        final Set<String> concreteIndices = new HashSet<>(expressions.size());
+        final Set<Index> concreteIndices = new HashSet<>(expressions.size());
         for (String expression : expressions) {
             AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(expression);
             if (aliasOrIndex == null) {
@@ -169,11 +206,11 @@ public class IndexNameExpressionResolver extends AbstractComponent {
                         throw new IndexClosedException(index.getIndex());
                     } else {
                         if (options.forbidClosedIndices() == false) {
-                            concreteIndices.add(index.getIndex().getName());
+                            concreteIndices.add(index.getIndex());
                         }
                     }
                 } else if (index.getState() == IndexMetaData.State.OPEN) {
-                    concreteIndices.add(index.getIndex().getName());
+                    concreteIndices.add(index.getIndex());
                 } else {
                     throw new IllegalStateException("index state [" + index.getState() + "] not supported");
                 }
@@ -185,7 +222,7 @@ public class IndexNameExpressionResolver extends AbstractComponent {
             infe.setResources("index_expression", indexExpressions);
             throw infe;
         }
-        return concreteIndices.toArray(new String[concreteIndices.size()]);
+        return concreteIndices.toArray(new Index[concreteIndices.size()]);
     }
 
     /**
@@ -200,9 +237,9 @@ public class IndexNameExpressionResolver extends AbstractComponent {
      * @throws IllegalArgumentException if the index resolution lead to more than one index
      * @return the concrete index obtained as a result of the index resolution
      */
-    public String concreteSingleIndex(ClusterState state, IndicesRequest request) {
+    public Index concreteSingleIndex(ClusterState state, IndicesRequest request) {
         String indexExpression = request.indices() != null && request.indices().length > 0 ? request.indices()[0] : null;
-        String[] indices = concreteIndices(state, request.indicesOptions(), indexExpression);
+        Index[] indices = concreteIndices(state, request.indicesOptions(), indexExpression);
         if (indices.length != 1) {
             throw new IllegalArgumentException("unable to return a single index as the index and options provided got resolved to multiple indices");
         }
@@ -395,7 +432,7 @@ public class IndexNameExpressionResolver extends AbstractComponent {
         if (routing != null) {
             Set<String> r = Strings.splitStringByCommaToSet(routing);
             Map<String, Set<String>> routings = new HashMap<>();
-            String[] concreteIndices = metaData.concreteAllIndices();
+            String[] concreteIndices = metaData.getConcreteAllIndices();
             for (String index : concreteIndices) {
                 routings.put(index, r);
             }
@@ -435,7 +472,7 @@ public class IndexNameExpressionResolver extends AbstractComponent {
      */
     boolean isPatternMatchingAllIndices(MetaData metaData, String[] indicesOrAliases, String[] concreteIndices) {
         // if we end up matching on all indices, check, if its a wildcard parameter, or a "-something" structure
-        if (concreteIndices.length == metaData.concreteAllIndices().length && indicesOrAliases.length > 0) {
+        if (concreteIndices.length == metaData.getConcreteAllIndices().length && indicesOrAliases.length > 0) {
 
             //we might have something like /-test1,+test1 that would identify all indices
             //or something like /-test1 with test1 index missing and IndicesOptions.lenient()
@@ -686,16 +723,16 @@ public class IndexNameExpressionResolver extends AbstractComponent {
         }
 
         private boolean isEmptyOrTrivialWildcard(List<String> expressions) {
-            return expressions.isEmpty() || (expressions.size() == 1 && (MetaData.ALL.equals(expressions.get(0))) || Regex.isMatchAllPattern(expressions.get(0)));
+            return expressions.isEmpty() || (expressions.size() == 1 && (MetaData.ALL.equals(expressions.get(0)) || Regex.isMatchAllPattern(expressions.get(0))));
         }
 
         private List<String> resolveEmptyOrTrivialWildcard(IndicesOptions options, MetaData metaData, boolean assertEmpty) {
             if (options.expandWildcardsOpen() && options.expandWildcardsClosed()) {
-                return Arrays.asList(metaData.concreteAllIndices());
+                return Arrays.asList(metaData.getConcreteAllIndices());
             } else if (options.expandWildcardsOpen()) {
-                return Arrays.asList(metaData.concreteAllOpenIndices());
+                return Arrays.asList(metaData.getConcreteAllOpenIndices());
             } else if (options.expandWildcardsClosed()) {
-                return Arrays.asList(metaData.concreteAllClosedIndices());
+                return Arrays.asList(metaData.getConcreteAllClosedIndices());
             } else {
                 assert assertEmpty : "Shouldn't end up here";
                 return Collections.emptyList();
@@ -867,7 +904,7 @@ public class IndexNameExpressionResolver extends AbstractComponent {
      * Returns <code>true</code> iff the given expression resolves to the given index name otherwise <code>false</code>
      */
     public final boolean matchesIndex(String indexName, String expression, ClusterState state) {
-        final String[] concreteIndices = concreteIndices(state, IndicesOptions.lenientExpandOpen(), expression);
+        final String[] concreteIndices = concreteIndexNames(state, IndicesOptions.lenientExpandOpen(), expression);
         for (String index : concreteIndices) {
             if (Regex.simpleMatch(index, indexName)) {
                 return true;
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
index a88f1609b9e..10b05c46657 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
@@ -84,20 +84,10 @@ public class MappingMetaData extends AbstractDiffable<MappingMetaData> {
 
         private static final FormatDateTimeFormatter EPOCH_MILLIS_PARSER = Joda.forPattern("epoch_millis");
 
-        public static String parseStringTimestamp(String timestampAsString, FormatDateTimeFormatter dateTimeFormatter,
-                                                  Version version) throws TimestampParsingException {
+        public static String parseStringTimestamp(String timestampAsString, FormatDateTimeFormatter dateTimeFormatter) throws TimestampParsingException {
             try {
-                // no need for unix timestamp parsing in 2.x
-                FormatDateTimeFormatter formatter = version.onOrAfter(Version.V_2_0_0_beta1) ? dateTimeFormatter : EPOCH_MILLIS_PARSER;
-                return Long.toString(formatter.parser().parseMillis(timestampAsString));
+                return Long.toString(dateTimeFormatter.parser().parseMillis(timestampAsString));
             } catch (RuntimeException e) {
-                if (version.before(Version.V_2_0_0_beta1)) {
-                    try {
-                        return Long.toString(dateTimeFormatter.parser().parseMillis(timestampAsString));
-                    } catch (RuntimeException e1) {
-                        throw new TimestampParsingException(timestampAsString, e1);
-                    }
-                }
                 throw new TimestampParsingException(timestampAsString, e);
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index d7dddb15984..db6871b0645 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -30,7 +30,7 @@ import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider;
-import org.elasticsearch.cluster.service.InternalClusterService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
@@ -41,6 +41,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.loader.SettingsLoader;
 import org.elasticsearch.common.xcontent.FromXContentBuilder;
@@ -50,6 +51,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.discovery.DiscoverySettings;
+import org.elasticsearch.gateway.MetaDataStateFormat;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.store.IndexStoreConfig;
@@ -139,7 +141,8 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
     }
 
 
-    public static final Setting<Boolean> SETTING_READ_ONLY_SETTING = Setting.boolSetting("cluster.blocks.read_only", false, true, Setting.Scope.CLUSTER);
+    public static final Setting<Boolean> SETTING_READ_ONLY_SETTING =
+        Setting.boolSetting("cluster.blocks.read_only", false, Property.Dynamic, Property.NodeScope);
 
     public static final ClusterBlock CLUSTER_READ_ONLY_BLOCK = new ClusterBlock(6, "cluster read-only (api)", false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA_WRITE));
 
@@ -151,6 +154,8 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
 
     public static final String CONTEXT_MODE_GATEWAY = XContentContext.GATEWAY.toString();
 
+    public static final String GLOBAL_STATE_FILE_PREFIX = "global-";
+
     private final String clusterUUID;
     private final long version;
 
@@ -230,7 +235,7 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
     public boolean equalsAliases(MetaData other) {
         for (ObjectCursor<IndexMetaData> cursor : other.indices().values()) {
             IndexMetaData otherIndex = cursor.value;
-            IndexMetaData thisIndex= index(otherIndex.getIndex());
+            IndexMetaData thisIndex = index(otherIndex.getIndex());
             if (thisIndex == null) {
                 return false;
             }
@@ -368,26 +373,14 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
     /**
      * Returns all the concrete indices.
      */
-    public String[] concreteAllIndices() {
-        return allIndices;
-    }
-
     public String[] getConcreteAllIndices() {
-        return concreteAllIndices();
-    }
-
-    public String[] concreteAllOpenIndices() {
-        return allOpenIndices;
+        return allIndices;
     }
 
     public String[] getConcreteAllOpenIndices() {
         return allOpenIndices;
     }
 
-    public String[] concreteAllClosedIndices() {
-        return allClosedIndices;
-    }
-
     public String[] getConcreteAllClosedIndices() {
         return allClosedIndices;
     }
@@ -455,7 +448,28 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
     }
 
     public IndexMetaData index(Index index) {
-        return index(index.getName());
+        IndexMetaData metaData = index(index.getName());
+        if (metaData != null && metaData.getIndexUUID().equals(index.getUUID())) {
+            return metaData;
+        }
+        return null;
+    }
+
+    /**
+     * Returns the {@link IndexMetaData} for this index.
+     * @throws IndexNotFoundException if no metadata for this index is found
+     */
+    public IndexMetaData getIndexSafe(Index index) {
+        IndexMetaData metaData = index(index.getName());
+        if (metaData != null) {
+            if(metaData.getIndexUUID().equals(index.getUUID())) {
+                return metaData;
+            }
+            throw new IndexNotFoundException(index,
+                new IllegalStateException("index uuid doesn't match expected: [" + index.getUUID()
+                    + "] but got: [" + metaData.getIndexUUID() +"]"));
+        }
+        throw new IndexNotFoundException(index);
     }
 
     public ImmutableOpenMap<String, IndexMetaData> indices() {
@@ -486,20 +500,13 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
         return (T) customs.get(type);
     }
 
-    public int totalNumberOfShards() {
+
+    public int getTotalNumberOfShards() {
         return this.totalNumberOfShards;
     }
 
-    public int getTotalNumberOfShards() {
-        return totalNumberOfShards();
-    }
-
-    public int numberOfShards() {
-        return this.numberOfShards;
-    }
-
     public int getNumberOfShards() {
-        return numberOfShards();
+        return this.numberOfShards;
     }
 
     /**
@@ -731,7 +738,7 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
                                     InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.getKey(),
                                     InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(),
                                     DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(),
-                                    InternalClusterService.CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.getKey()));
+            ClusterService.CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.getKey()));
 
     /** As of 2.0 we require units for time and byte-sized settings.  This methods adds default units to any cluster settings that don't
      *  specify a unit. */
@@ -779,9 +786,9 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
                     metaData.getIndices(),
                     metaData.getTemplates(),
                     metaData.getCustoms(),
-                    metaData.concreteAllIndices(),
-                    metaData.concreteAllOpenIndices(),
-                    metaData.concreteAllClosedIndices(),
+                    metaData.getConcreteAllIndices(),
+                    metaData.getConcreteAllOpenIndices(),
+                    metaData.getConcreteAllClosedIndices(),
                     metaData.getAliasAndIndexLookup());
         } else {
             // No changes:
@@ -842,6 +849,19 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
             return indices.get(index);
         }
 
+        public IndexMetaData getSafe(Index index) {
+            IndexMetaData indexMetaData = get(index.getName());
+            if (indexMetaData != null) {
+                if(indexMetaData.getIndexUUID().equals(index.getUUID())) {
+                    return indexMetaData;
+                }
+                throw new IndexNotFoundException(index,
+                    new IllegalStateException("index uuid doesn't match expected: [" + index.getUUID()
+                        + "] but got: [" + indexMetaData.getIndexUUID() +"]"));
+            }
+            throw new IndexNotFoundException(index);
+        }
+
         public Builder remove(String index) {
             indices.remove(index);
             return this;
@@ -1143,4 +1163,28 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
             return PROTO.readFrom(in);
         }
     }
+
+    private final static ToXContent.Params FORMAT_PARAMS;
+    static {
+        Map<String, String> params = new HashMap<>(2);
+        params.put("binary", "true");
+        params.put(MetaData.CONTEXT_MODE_PARAM, MetaData.CONTEXT_MODE_GATEWAY);
+        FORMAT_PARAMS = new MapParams(params);
+    }
+
+    /**
+     * State format for {@link MetaData} to write to and load from disk
+     */
+    public final static MetaDataStateFormat<MetaData> FORMAT = new MetaDataStateFormat<MetaData>(XContentType.SMILE, GLOBAL_STATE_FILE_PREFIX) {
+
+        @Override
+        public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
+            Builder.toXContent(state, builder, FORMAT_PARAMS);
+        }
+
+        @Override
+        public MetaData fromXContent(XContentParser parser) throws IOException {
+            return Builder.fromXContent(parser);
+        }
+    };
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
index 62f3ad802a0..e0db19cb516 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
@@ -28,7 +28,6 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.block.ClusterBlock;
@@ -39,6 +38,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.ValidationException;
@@ -53,6 +53,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.mapper.DocumentMapper;
@@ -188,7 +189,7 @@ public class MetaDataCreateIndexService extends AbstractComponent {
 
                     @Override
                     public ClusterState execute(ClusterState currentState) throws Exception {
-                        boolean indexCreated = false;
+                        Index createdIndex = null;
                         String removalReason = null;
                         try {
                             validate(request, currentState);
@@ -308,10 +309,9 @@ public class MetaDataCreateIndexService extends AbstractComponent {
                             // Set up everything, now locally create the index to see that things are ok, and apply
                             final IndexMetaData tmpImd = IndexMetaData.builder(request.index()).settings(actualIndexSettings).build();
                             // create the index here (on the master) to validate it can be created, as well as adding the mapping
-                            indicesService.createIndex(nodeServicesProvider, tmpImd, Collections.emptyList());
-                            indexCreated = true;
+                            final IndexService indexService = indicesService.createIndex(nodeServicesProvider, tmpImd, Collections.emptyList());
+                            createdIndex = indexService.index();
                             // now add the mappings
-                            IndexService indexService = indicesService.indexServiceSafe(request.index());
                             MapperService mapperService = indexService.mapperService();
                             // first, add the default mapping
                             if (mappings.containsKey(MapperService.DEFAULT_MAPPING)) {
@@ -415,9 +415,9 @@ public class MetaDataCreateIndexService extends AbstractComponent {
                             removalReason = "cleaning up after validating index on master";
                             return updatedState;
                         } finally {
-                            if (indexCreated) {
+                            if (createdIndex != null) {
                                 // Index was already partially created - need to clean up
-                                indicesService.removeIndex(request.index(), removalReason != null ? removalReason : "failed to create index");
+                                indicesService.removeIndex(createdIndex, removalReason != null ? removalReason : "failed to create index");
                             }
                         }
                     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
index 54c014fb4ed..5e6d35aacfe 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.action.support.master.MasterNodeRequest;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction;
@@ -28,20 +27,22 @@ import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
-import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.threadpool.ThreadPool;
 
-import java.util.Arrays;
-import java.util.Collection;
+import java.util.Set;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
 
 /**
  *
@@ -67,10 +68,9 @@ public class MetaDataDeleteIndexService extends AbstractComponent {
     }
 
     public void deleteIndices(final Request request, final Listener userListener) {
-        Collection<String> indices = Arrays.asList(request.indices);
         final DeleteIndexListener listener = new DeleteIndexListener(userListener);
 
-        clusterService.submitStateUpdateTask("delete-index " + indices, new ClusterStateUpdateTask(Priority.URGENT) {
+        clusterService.submitStateUpdateTask("delete-index " + request.indices, new ClusterStateUpdateTask(Priority.URGENT) {
 
             @Override
             public TimeValue timeout() {
@@ -84,20 +84,21 @@ public class MetaDataDeleteIndexService extends AbstractComponent {
 
             @Override
             public ClusterState execute(final ClusterState currentState) {
+                final MetaData meta = currentState.metaData();
+                final Set<IndexMetaData> metaDatas = request.indices.stream().map(i -> meta.getIndexSafe(i)).collect(Collectors.toSet());
+                // Check if index deletion conflicts with any running snapshots
+                SnapshotsService.checkIndexDeletion(currentState, metaDatas);
+                final Set<Index> indices = request.indices;
                 RoutingTable.Builder routingTableBuilder = RoutingTable.builder(currentState.routingTable());
-                MetaData.Builder metaDataBuilder = MetaData.builder(currentState.metaData());
+                MetaData.Builder metaDataBuilder = MetaData.builder(meta);
                 ClusterBlocks.Builder clusterBlocksBuilder = ClusterBlocks.builder().blocks(currentState.blocks());
 
-                for (final String index: indices) {
-                    if (!currentState.metaData().hasConcreteIndex(index)) {
-                        throw new IndexNotFoundException(index);
-                    }
-
+                for (final Index index : indices) {
+                    String indexName = index.getName();
                     logger.debug("[{}] deleting index", index);
-
-                    routingTableBuilder.remove(index);
-                    clusterBlocksBuilder.removeIndexBlocks(index);
-                    metaDataBuilder.remove(index);
+                    routingTableBuilder.remove(indexName);
+                    clusterBlocksBuilder.removeIndexBlocks(indexName);
+                    metaDataBuilder.remove(indexName);
                 }
                 // wait for events from all nodes that it has been removed from their respective metadata...
                 int count = currentState.nodes().size();
@@ -108,7 +109,7 @@ public class MetaDataDeleteIndexService extends AbstractComponent {
                 // this listener will be notified once we get back a notification based on the cluster state change below.
                 final NodeIndexDeletedAction.Listener nodeIndexDeleteListener = new NodeIndexDeletedAction.Listener() {
                     @Override
-                    public void onNodeIndexDeleted(String deleted, String nodeId) {
+                    public void onNodeIndexDeleted(Index deleted, String nodeId) {
                         if (indices.contains(deleted)) {
                             if (counter.decrementAndGet() == 0) {
                                 listener.onResponse(new Response(true));
@@ -118,7 +119,7 @@ public class MetaDataDeleteIndexService extends AbstractComponent {
                     }
 
                     @Override
-                    public void onNodeIndexStoreDeleted(String deleted, String nodeId) {
+                    public void onNodeIndexStoreDeleted(Index deleted, String nodeId) {
                         if (indices.contains(deleted)) {
                             if (counter.decrementAndGet() == 0) {
                                 listener.onResponse(new Response(true));
@@ -183,12 +184,12 @@ public class MetaDataDeleteIndexService extends AbstractComponent {
 
     public static class Request {
 
-        final String[] indices;
+        final Set<Index> indices;
 
         TimeValue timeout = TimeValue.timeValueSeconds(10);
         TimeValue masterTimeout = MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT;
 
-        public Request(String[] indices) {
+        public Request(Set<Index> indices) {
             this.indices = indices;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java
index 52154bd2c04..e39b86a1611 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java
@@ -23,14 +23,15 @@ import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesClusterStateUpdateRequest;
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.NodeServicesProvider;
@@ -74,7 +75,7 @@ public class MetaDataIndexAliasesService extends AbstractComponent {
 
             @Override
             public ClusterState execute(final ClusterState currentState) {
-                List<String> indicesToClose = new ArrayList<>();
+                List<Index> indicesToClose = new ArrayList<>();
                 Map<String, IndexService> indices = new HashMap<>();
                 try {
                     for (AliasAction aliasAction : request.actions()) {
@@ -112,7 +113,7 @@ public class MetaDataIndexAliasesService extends AbstractComponent {
                                             logger.warn("[{}] failed to temporary create in order to apply alias action", e, indexMetaData.getIndex());
                                             continue;
                                         }
-                                        indicesToClose.add(indexMetaData.getIndex().getName());
+                                        indicesToClose.add(indexMetaData.getIndex());
                                     }
                                     indices.put(indexMetaData.getIndex().getName(), indexService);
                                 }
@@ -153,7 +154,7 @@ public class MetaDataIndexAliasesService extends AbstractComponent {
                     }
                     return currentState;
                 } finally {
-                    for (String index : indicesToClose) {
+                    for (Index index : indicesToClose) {
                         indicesService.removeIndex(index, "created for alias processing");
                     }
                 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java
index 6639f9bdbd6..e6e7084e4d9 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java
@@ -19,14 +19,12 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.close.CloseIndexClusterStateUpdateRequest;
 import org.elasticsearch.action.admin.indices.open.OpenIndexClusterStateUpdateRequest;
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.RestoreInProgress;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
@@ -34,13 +32,17 @@ import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.NodeServicesProvider;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.snapshots.RestoreService;
+import org.elasticsearch.snapshots.SnapshotsService;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -60,10 +62,16 @@ public class MetaDataIndexStateService extends AbstractComponent {
     private final AllocationService allocationService;
 
     private final MetaDataIndexUpgradeService metaDataIndexUpgradeService;
+    private final NodeServicesProvider nodeServiceProvider;
+    private final IndicesService indicesService;
 
     @Inject
-    public MetaDataIndexStateService(Settings settings, ClusterService clusterService, AllocationService allocationService, MetaDataIndexUpgradeService metaDataIndexUpgradeService) {
+    public MetaDataIndexStateService(Settings settings, ClusterService clusterService, AllocationService allocationService,
+                                     MetaDataIndexUpgradeService metaDataIndexUpgradeService,
+                                     NodeServicesProvider nodeServicesProvider, IndicesService indicesService) {
         super(settings);
+        this.nodeServiceProvider = nodeServicesProvider;
+        this.indicesService = indicesService;
         this.clusterService = clusterService;
         this.allocationService = allocationService;
         this.metaDataIndexUpgradeService = metaDataIndexUpgradeService;
@@ -83,15 +91,11 @@ public class MetaDataIndexStateService extends AbstractComponent {
 
             @Override
             public ClusterState execute(ClusterState currentState) {
-                Set<String> indicesToClose = new HashSet<>();
-                for (String index : request.indices()) {
-                    IndexMetaData indexMetaData = currentState.metaData().index(index);
-                    if (indexMetaData == null) {
-                        throw new IndexNotFoundException(index);
-                    }
-
+                Set<IndexMetaData> indicesToClose = new HashSet<>();
+                for (Index index : request.indices()) {
+                    final IndexMetaData indexMetaData = currentState.metaData().getIndexSafe(index);
                     if (indexMetaData.getState() != IndexMetaData.State.CLOSE) {
-                        indicesToClose.add(index);
+                        indicesToClose.add(indexMetaData);
                     }
                 }
 
@@ -99,43 +103,26 @@ public class MetaDataIndexStateService extends AbstractComponent {
                     return currentState;
                 }
 
-                // Check if any of the indices to be closed are currently being restored from a snapshot and fail closing if such an index
-                // is found as closing an index that is being restored makes the index unusable (it cannot be recovered).
-                RestoreInProgress restore = currentState.custom(RestoreInProgress.TYPE);
-                if (restore != null) {
-                    Set<String> indicesToFail = null;
-                    for (RestoreInProgress.Entry entry : restore.entries()) {
-                        for (ObjectObjectCursor<ShardId, RestoreInProgress.ShardRestoreStatus> shard : entry.shards()) {
-                            if (!shard.value.state().completed()) {
-                                if (indicesToClose.contains(shard.key.getIndexName())) {
-                                    if (indicesToFail == null) {
-                                        indicesToFail = new HashSet<>();
-                                    }
-                                    indicesToFail.add(shard.key.getIndexName());
-                                }
-                            }
-                        }
-                    }
-                    if (indicesToFail != null) {
-                        throw new IllegalArgumentException("Cannot close indices that are being restored: " + indicesToFail);
-                    }
-                }
-
+                // Check if index closing conflicts with any running restores
+                RestoreService.checkIndexClosing(currentState, indicesToClose);
+                // Check if index closing conflicts with any running snapshots
+                SnapshotsService.checkIndexClosing(currentState, indicesToClose);
                 logger.info("closing indices [{}]", indicesAsString);
 
                 MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
                 ClusterBlocks.Builder blocksBuilder = ClusterBlocks.builder()
                         .blocks(currentState.blocks());
-                for (String index : indicesToClose) {
-                    mdBuilder.put(IndexMetaData.builder(currentState.metaData().index(index)).state(IndexMetaData.State.CLOSE));
-                    blocksBuilder.addIndexBlock(index, INDEX_CLOSED_BLOCK);
+                for (IndexMetaData openIndexMetadata : indicesToClose) {
+                    final String indexName = openIndexMetadata.getIndex().getName();
+                    mdBuilder.put(IndexMetaData.builder(openIndexMetadata).state(IndexMetaData.State.CLOSE));
+                    blocksBuilder.addIndexBlock(indexName, INDEX_CLOSED_BLOCK);
                 }
 
                 ClusterState updatedState = ClusterState.builder(currentState).metaData(mdBuilder).blocks(blocksBuilder).build();
 
                 RoutingTable.Builder rtBuilder = RoutingTable.builder(currentState.routingTable());
-                for (String index : indicesToClose) {
-                    rtBuilder.remove(index);
+                for (IndexMetaData index : indicesToClose) {
+                    rtBuilder.remove(index.getIndex().getName());
                 }
 
                 RoutingAllocation.Result routingResult = allocationService.reroute(
@@ -161,14 +148,11 @@ public class MetaDataIndexStateService extends AbstractComponent {
 
             @Override
             public ClusterState execute(ClusterState currentState) {
-                List<String> indicesToOpen = new ArrayList<>();
-                for (String index : request.indices()) {
-                    IndexMetaData indexMetaData = currentState.metaData().index(index);
-                    if (indexMetaData == null) {
-                        throw new IndexNotFoundException(index);
-                    }
+                List<IndexMetaData> indicesToOpen = new ArrayList<>();
+                for (Index index : request.indices()) {
+                    final IndexMetaData indexMetaData = currentState.metaData().getIndexSafe(index);
                     if (indexMetaData.getState() != IndexMetaData.State.OPEN) {
-                        indicesToOpen.add(index);
+                        indicesToOpen.add(indexMetaData);
                     }
                 }
 
@@ -181,20 +165,27 @@ public class MetaDataIndexStateService extends AbstractComponent {
                 MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
                 ClusterBlocks.Builder blocksBuilder = ClusterBlocks.builder()
                         .blocks(currentState.blocks());
-                for (String index : indicesToOpen) {
-                    IndexMetaData indexMetaData = IndexMetaData.builder(currentState.metaData().index(index)).state(IndexMetaData.State.OPEN).build();
+                for (IndexMetaData closedMetaData : indicesToOpen) {
+                    final String indexName = closedMetaData.getIndex().getName();
+                    IndexMetaData indexMetaData = IndexMetaData.builder(closedMetaData).state(IndexMetaData.State.OPEN).build();
                     // The index might be closed because we couldn't import it due to old incompatible version
                     // We need to check that this index can be upgraded to the current version
                     indexMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(indexMetaData);
+                    try {
+                        indicesService.verifyIndexMetadata(nodeServiceProvider, indexMetaData);
+                    } catch (Exception e) {
+                        throw new ElasticsearchException("Failed to verify index " + indexMetaData.getIndex(), e);
+                    }
+
                     mdBuilder.put(indexMetaData, true);
-                    blocksBuilder.removeIndexBlock(index, INDEX_CLOSED_BLOCK);
+                    blocksBuilder.removeIndexBlock(indexName, INDEX_CLOSED_BLOCK);
                 }
 
                 ClusterState updatedState = ClusterState.builder(currentState).metaData(mdBuilder).blocks(blocksBuilder).build();
 
                 RoutingTable.Builder rtBuilder = RoutingTable.builder(updatedState.routingTable());
-                for (String index : indicesToOpen) {
-                    rtBuilder.addAsFromCloseToOpen(updatedState.metaData().index(index));
+                for (IndexMetaData index : indicesToOpen) {
+                    rtBuilder.addAsFromCloseToOpen(updatedState.metaData().getIndexSafe(index.getIndex()));
                 }
 
                 RoutingAllocation.Result routingResult = allocationService.reroute(
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
index da2fc064dc4..1206185a609 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
@@ -21,9 +21,9 @@ package org.elasticsearch.cluster.metadata;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.ValidationException;
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
index 8bbd6f09d7e..3101a2c04cc 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
@@ -20,15 +20,12 @@ package org.elasticsearch.cluster.metadata;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.misc.IndexMergeTool;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.IndexScopedSettings;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.MergePolicyConfig;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.MapperService;
@@ -36,10 +33,7 @@ import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.mapper.MapperRegistry;
 
 import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
 
-import static java.util.Collections.unmodifiableSet;
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
 
 /**
@@ -53,13 +47,13 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet;
 public class MetaDataIndexUpgradeService extends AbstractComponent {
 
     private final MapperRegistry mapperRegistry;
-    private final IndexScopedSettings indexScopedSettigns;
+    private final IndexScopedSettings indexScopedSettings;
 
     @Inject
     public MetaDataIndexUpgradeService(Settings settings, MapperRegistry mapperRegistry, IndexScopedSettings indexScopedSettings) {
         super(settings);
         this.mapperRegistry = mapperRegistry;
-        this.indexScopedSettigns = indexScopedSettings;
+        this.indexScopedSettings = indexScopedSettings;
     }
 
     /**
@@ -182,39 +176,13 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
         }
     }
 
-    private static final String ARCHIVED_SETTINGS_PREFIX = "archived.";
-
     IndexMetaData archiveBrokenIndexSettings(IndexMetaData indexMetaData) {
-        Settings settings = indexMetaData.getSettings();
-        Settings.Builder builder = Settings.builder();
-        boolean changed = false;
-        for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
-            try {
-                Setting<?> setting = indexScopedSettigns.get(entry.getKey());
-                if (setting != null) {
-                    setting.get(settings);
-                    builder.put(entry.getKey(), entry.getValue());
-                } else {
-                    if (indexScopedSettigns.isPrivateSetting(entry.getKey()) || entry.getKey().startsWith(ARCHIVED_SETTINGS_PREFIX)) {
-                        builder.put(entry.getKey(), entry.getValue());
-                    } else {
-                        changed = true;
-                        logger.warn("[{}] found unknown index setting: {} value: {} - archiving", indexMetaData.getIndex(), entry.getKey(), entry.getValue());
-                        // we put them back in here such that tools can check from the outside if there are any indices with broken settings. The setting can remain there
-                        // but we want users to be aware that some of their setting are broken and they can research why and what they need to do to replace them.
-                        builder.put(ARCHIVED_SETTINGS_PREFIX + entry.getKey(), entry.getValue());
-                    }
-                }
-            } catch (IllegalArgumentException ex) {
-                changed = true;
-                logger.warn("[{}] found invalid index setting: {} value: {} - archiving",ex, indexMetaData.getIndex(), entry.getKey(), entry.getValue());
-                // we put them back in here such that tools can check from the outside if there are any indices with broken settings. The setting can remain there
-                // but we want users to be aware that some of their setting sare broken and they can research why and what they need to do to replace them.
-                builder.put(ARCHIVED_SETTINGS_PREFIX + entry.getKey(), entry.getValue());
-            }
+        final Settings settings = indexMetaData.getSettings();
+        final Settings upgrade = indexScopedSettings.archiveUnknownOrBrokenSettings(settings);
+        if (upgrade != settings) {
+            return IndexMetaData.builder(indexMetaData).settings(upgrade).build();
+        } else {
+            return indexMetaData;
         }
-
-        return changed ? IndexMetaData.builder(indexMetaData).settings(builder.build()).build() : indexMetaData;
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
index c06a5cc7c1c..cafdc4581a1 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
@@ -23,27 +23,28 @@ import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingClusterStateUpdateRequest;
 import org.elasticsearch.cluster.AckedClusterStateTaskListener;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateTaskConfig;
 import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.percolator.PercolatorFieldMapper;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.InvalidTypeNameException;
-import org.elasticsearch.percolator.PercolatorService;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -112,13 +113,13 @@ public class MetaDataMappingService extends AbstractComponent {
         MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
 
         for (Map.Entry<String, List<RefreshTask>> entry : tasksPerIndex.entrySet()) {
-            String index = entry.getKey();
-            IndexMetaData indexMetaData = mdBuilder.get(index);
+            IndexMetaData indexMetaData = mdBuilder.get(entry.getKey());
             if (indexMetaData == null) {
                 // index got deleted on us, ignore...
-                logger.debug("[{}] ignoring tasks - index meta data doesn't exist", index);
+                logger.debug("[{}] ignoring tasks - index meta data doesn't exist", entry.getKey());
                 continue;
             }
+            final Index index = indexMetaData.getIndex();
             // the tasks lists to iterate over, filled with the list of mapping tasks, trying to keep
             // the latest (based on order) update mapping one per node
             List<RefreshTask> allIndexTasks = entry.getValue();
@@ -127,7 +128,7 @@ public class MetaDataMappingService extends AbstractComponent {
                 if (indexMetaData.isSameUUID(task.indexUUID)) {
                     hasTaskWithRightUUID = true;
                 } else {
-                    logger.debug("[{}] ignoring task [{}] - index meta data doesn't match task uuid", index, task);
+                    logger.debug("{} ignoring task [{}] - index meta data doesn't match task uuid", index, task);
                 }
             }
             if (hasTaskWithRightUUID == false) {
@@ -136,7 +137,7 @@ public class MetaDataMappingService extends AbstractComponent {
 
             // construct the actual index if needed, and make sure the relevant mappings are there
             boolean removeIndex = false;
-            IndexService indexService = indicesService.indexService(index);
+            IndexService indexService = indicesService.indexService(indexMetaData.getIndex());
             if (indexService == null) {
                 // we need to create the index here, and add the current mapping to it, so we can merge
                 indexService = indicesService.createIndex(nodeServicesProvider, indexMetaData, Collections.emptyList());
@@ -208,38 +209,38 @@ public class MetaDataMappingService extends AbstractComponent {
 
     class PutMappingExecutor implements ClusterStateTaskExecutor<PutMappingClusterStateUpdateRequest> {
         @Override
-        public BatchResult<PutMappingClusterStateUpdateRequest> execute(ClusterState currentState, List<PutMappingClusterStateUpdateRequest> tasks) throws Exception {
-            Set<String> indicesToClose = new HashSet<>();
+        public BatchResult<PutMappingClusterStateUpdateRequest> execute(ClusterState currentState,
+                                                                        List<PutMappingClusterStateUpdateRequest> tasks) throws Exception {
+            Set<Index> indicesToClose = new HashSet<>();
             BatchResult.Builder<PutMappingClusterStateUpdateRequest> builder = BatchResult.builder();
             try {
                 // precreate incoming indices;
                 for (PutMappingClusterStateUpdateRequest request : tasks) {
-                    // failures here mean something is broken with our cluster state - fail all tasks by letting exceptions bubble up
-                    for (String index : request.indices()) {
-                        final IndexMetaData indexMetaData = currentState.metaData().index(index);
-                        if (indexMetaData != null  && indicesService.hasIndex(index) == false) {
-                            // if we don't have the index, we will throw exceptions later;
-                            indicesToClose.add(index);
-                            IndexService indexService = indicesService.createIndex(nodeServicesProvider, indexMetaData, Collections.emptyList());
-                            // add mappings for all types, we need them for cross-type validation
-                            for (ObjectCursor<MappingMetaData> mapping : indexMetaData.getMappings().values()) {
-                                indexService.mapperService().merge(mapping.value.type(), mapping.value.source(), MapperService.MergeReason.MAPPING_RECOVERY, request.updateAllTypes());
+                    try {
+                        for (Index index : request.indices()) {
+                            final IndexMetaData indexMetaData = currentState.metaData().getIndexSafe(index);
+                            if (indicesService.hasIndex(indexMetaData.getIndex()) == false) {
+                                // if the index does not exists we create it once, add all types to the mapper service and
+                                // close it later once we are done with mapping update
+                                indicesToClose.add(indexMetaData.getIndex());
+                                IndexService indexService = indicesService.createIndex(nodeServicesProvider, indexMetaData,
+                                    Collections.emptyList());
+                                // add mappings for all types, we need them for cross-type validation
+                                for (ObjectCursor<MappingMetaData> mapping : indexMetaData.getMappings().values()) {
+                                    indexService.mapperService().merge(mapping.value.type(), mapping.value.source(),
+                                        MapperService.MergeReason.MAPPING_RECOVERY, request.updateAllTypes());
+                                }
                             }
                         }
-                    }
-                }
-                for (PutMappingClusterStateUpdateRequest request : tasks) {
-                    try {
                         currentState = applyRequest(currentState, request);
                         builder.success(request);
                     } catch (Throwable t) {
                         builder.failure(request, t);
                     }
                 }
-
                 return builder.build(currentState);
             } finally {
-                for (String index : indicesToClose) {
+                for (Index index : indicesToClose) {
                     indicesService.removeIndex(index, "created for mapping processing");
                 }
             }
@@ -248,8 +249,17 @@ public class MetaDataMappingService extends AbstractComponent {
         private ClusterState applyRequest(ClusterState currentState, PutMappingClusterStateUpdateRequest request) throws IOException {
             String mappingType = request.type();
             CompressedXContent mappingUpdateSource = new CompressedXContent(request.source());
-            for (String index : request.indices()) {
+            final MetaData metaData = currentState.metaData();
+            final List<Tuple<IndexService, IndexMetaData>> updateList = new ArrayList<>();
+            for (Index index : request.indices()) {
                 IndexService indexService = indicesService.indexServiceSafe(index);
+                // IMPORTANT: always get the metadata from the state since it get's batched
+                // and if we pull it from the indexService we might miss an update etc.
+                final IndexMetaData indexMetaData = currentState.getMetaData().getIndexSafe(index);
+
+                // this is paranoia... just to be sure we use the exact same indexService and metadata tuple on the update that
+                // we used for the validation, it makes this mechanism little less scary (a little)
+                updateList.add(new Tuple<>(indexService, indexMetaData));
                 // try and parse it (no need to add it here) so we can bail early in case of parsing exception
                 DocumentMapper newMapper;
                 DocumentMapper existingMapper = indexService.mapperService().documentMapper(request.type());
@@ -270,7 +280,6 @@ public class MetaDataMappingService extends AbstractComponent {
                         // and a put mapping api call, so we don't which type did exist before.
                         // Also the order of the mappings may be backwards.
                         if (newMapper.parentFieldMapper().active()) {
-                            IndexMetaData indexMetaData = currentState.metaData().index(index);
                             for (ObjectCursor<MappingMetaData> mapping : indexMetaData.getMappings().values()) {
                                 if (newMapper.parentFieldMapper().type().equals(mapping.value.type())) {
                                     throw new IllegalArgumentException("can't add a _parent field that points to an already existing type");
@@ -287,17 +296,16 @@ public class MetaDataMappingService extends AbstractComponent {
             }
             assert mappingType != null;
 
-            if (!MapperService.DEFAULT_MAPPING.equals(mappingType) && !PercolatorService.TYPE_NAME.equals(mappingType) && mappingType.charAt(0) == '_') {
+            if (!MapperService.DEFAULT_MAPPING.equals(mappingType) && !PercolatorFieldMapper.TYPE_NAME.equals(mappingType) && mappingType.charAt(0) == '_') {
                 throw new InvalidTypeNameException("Document mapping type name can't start with '_'");
             }
-            MetaData.Builder builder = MetaData.builder(currentState.metaData());
-            for (String index : request.indices()) {
+            MetaData.Builder builder = MetaData.builder(metaData);
+            for (Tuple<IndexService, IndexMetaData> toUpdate : updateList) {
                 // do the actual merge here on the master, and update the mapping source
-                IndexService indexService = indicesService.indexService(index);
-                if (indexService == null) {
-                    continue;
-                }
-
+                // we use the exact same indexService and metadata we used to validate above here to actually apply the update
+                final IndexService indexService = toUpdate.v1();
+                final IndexMetaData indexMetaData = toUpdate.v2();
+                final Index index = indexMetaData.getIndex();
                 CompressedXContent existingSource = null;
                 DocumentMapper existingMapper = indexService.mapperService().documentMapper(mappingType);
                 if (existingMapper != null) {
@@ -312,24 +320,20 @@ public class MetaDataMappingService extends AbstractComponent {
                     } else {
                         // use the merged mapping source
                         if (logger.isDebugEnabled()) {
-                            logger.debug("[{}] update_mapping [{}] with source [{}]", index, mergedMapper.type(), updatedSource);
+                            logger.debug("{} update_mapping [{}] with source [{}]", index, mergedMapper.type(), updatedSource);
                         } else if (logger.isInfoEnabled()) {
-                            logger.info("[{}] update_mapping [{}]", index, mergedMapper.type());
+                            logger.info("{} update_mapping [{}]", index, mergedMapper.type());
                         }
 
                     }
                 } else {
                     if (logger.isDebugEnabled()) {
-                        logger.debug("[{}] create_mapping [{}] with source [{}]", index, mappingType, updatedSource);
+                        logger.debug("{} create_mapping [{}] with source [{}]", index, mappingType, updatedSource);
                     } else if (logger.isInfoEnabled()) {
-                        logger.info("[{}] create_mapping [{}]", index, mappingType);
+                        logger.info("{} create_mapping [{}]", index, mappingType);
                     }
                 }
 
-                IndexMetaData indexMetaData = currentState.metaData().index(index);
-                if (indexMetaData == null) {
-                    throw new IndexNotFoundException(index);
-                }
                 IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(indexMetaData);
                 // Mapping updates on a single type may have side-effects on other types so we need to
                 // update mapping metadata on all types
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
index 2d7ba4c3c05..c925b43056b 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
@@ -23,10 +23,8 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsClusterStateUpdateRequest;
 import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeSettingsClusterStateUpdateRequest;
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
@@ -35,6 +33,7 @@ import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
@@ -43,7 +42,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.Index;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -86,7 +85,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
         // we will want to know this for translating "all" to a number
         final int dataNodeCount = event.state().nodes().dataNodes().size();
 
-        Map<Integer, List<String>> nrReplicasChanged = new HashMap<>();
+        Map<Integer, List<Index>> nrReplicasChanged = new HashMap<>();
         // we need to do this each time in case it was changed by update settings
         for (final IndexMetaData indexMetaData : event.state().metaData()) {
             AutoExpandReplicas autoExpandReplicas = IndexMetaData.INDEX_AUTO_EXPAND_REPLICAS_SETTING.get(indexMetaData.getSettings());
@@ -117,7 +116,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
                         nrReplicasChanged.put(numberOfReplicas, new ArrayList<>());
                     }
 
-                    nrReplicasChanged.get(numberOfReplicas).add(indexMetaData.getIndex().getName());
+                    nrReplicasChanged.get(numberOfReplicas).add(indexMetaData.getIndex());
                 }
             }
         }
@@ -126,25 +125,25 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
             // update settings and kick of a reroute (implicit) for them to take effect
             for (final Integer fNumberOfReplicas : nrReplicasChanged.keySet()) {
                 Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, fNumberOfReplicas).build();
-                final List<String> indices = nrReplicasChanged.get(fNumberOfReplicas);
+                final List<Index> indices = nrReplicasChanged.get(fNumberOfReplicas);
 
                 UpdateSettingsClusterStateUpdateRequest updateRequest = new UpdateSettingsClusterStateUpdateRequest()
-                        .indices(indices.toArray(new String[indices.size()])).settings(settings)
+                        .indices(indices.toArray(new Index[indices.size()])).settings(settings)
                         .ackTimeout(TimeValue.timeValueMillis(0)) //no need to wait for ack here
                         .masterNodeTimeout(TimeValue.timeValueMinutes(10));
 
                 updateSettings(updateRequest, new ActionListener<ClusterStateUpdateResponse>() {
                     @Override
                     public void onResponse(ClusterStateUpdateResponse response) {
-                        for (String index : indices) {
-                            logger.info("[{}] auto expanded replicas to [{}]", index, fNumberOfReplicas);
+                        for (Index index : indices) {
+                            logger.info("{} auto expanded replicas to [{}]", index, fNumberOfReplicas);
                         }
                     }
 
                     @Override
                     public void onFailure(Throwable t) {
-                        for (String index : indices) {
-                            logger.warn("[{}] fail to auto expand replicas to [{}]", index, fNumberOfReplicas);
+                        for (Index index : indices) {
+                            logger.warn("{} fail to auto expand replicas to [{}]", index, fNumberOfReplicas);
                         }
                     }
                 });
@@ -177,6 +176,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
         final Settings skippedSettigns = skipppedSettings.build();
         final Settings closedSettings = settingsForClosedIndices.build();
         final Settings openSettings = settingsForOpenIndices.build();
+        final boolean preserveExisting = request.isPreserveExisting();
 
         clusterService.submitStateUpdateTask("update-settings",
                 new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, listener) {
@@ -188,16 +188,19 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
 
             @Override
             public ClusterState execute(ClusterState currentState) {
-                String[] actualIndices = indexNameExpressionResolver.concreteIndices(currentState, IndicesOptions.strictExpand(), request.indices());
                 RoutingTable.Builder routingTableBuilder = RoutingTable.builder(currentState.routingTable());
                 MetaData.Builder metaDataBuilder = MetaData.builder(currentState.metaData());
 
                 // allow to change any settings to a close index, and only allow dynamic settings to be changed
                 // on an open index
-                Set<String> openIndices = new HashSet<>();
-                Set<String> closeIndices = new HashSet<>();
-                for (String index : actualIndices) {
-                    if (currentState.metaData().index(index).getState() == IndexMetaData.State.OPEN) {
+                Set<Index> openIndices = new HashSet<>();
+                Set<Index> closeIndices = new HashSet<>();
+                final String[] actualIndices = new String[request.indices().length];
+                for (int i = 0; i < request.indices().length; i++) {
+                    Index index = request.indices()[i];
+                    actualIndices[i] = index.getName();
+                    final IndexMetaData metaData = currentState.metaData().getIndexSafe(index);
+                    if (metaData.getState() == IndexMetaData.State.OPEN) {
                         openIndices.add(index);
                     } else {
                         closeIndices.add(index);
@@ -206,20 +209,20 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
 
                 if (closeIndices.size() > 0 && closedSettings.get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS) != null) {
                     throw new IllegalArgumentException(String.format(Locale.ROOT,
-                            "Can't update [%s] on closed indices [%s] - can leave index in an unopenable state", IndexMetaData.SETTING_NUMBER_OF_REPLICAS,
+                            "Can't update [%s] on closed indices %s - can leave index in an unopenable state", IndexMetaData.SETTING_NUMBER_OF_REPLICAS,
                             closeIndices
                     ));
                 }
                 if (!skippedSettigns.getAsMap().isEmpty() && !openIndices.isEmpty()) {
                     throw new IllegalArgumentException(String.format(Locale.ROOT,
-                            "Can't update non dynamic settings[%s] for open indices [%s]",
+                            "Can't update non dynamic settings [%s] for open indices %s",
                             skippedSettigns.getAsMap().keySet(),
                             openIndices
                     ));
                 }
 
                 int updatedNumberOfReplicas = openSettings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, -1);
-                if (updatedNumberOfReplicas != -1) {
+                if (updatedNumberOfReplicas != -1 && preserveExisting == false) {
                     routingTableBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
                     metaDataBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
                     logger.info("updating number_of_replicas to [{}] for indices {}", updatedNumberOfReplicas, actualIndices);
@@ -232,28 +235,28 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
                 maybeUpdateClusterBlock(actualIndices, blocks, IndexMetaData.INDEX_READ_BLOCK, IndexMetaData.INDEX_BLOCKS_READ_SETTING, openSettings);
 
                 if (!openIndices.isEmpty()) {
-                    for (String index : openIndices) {
-                        IndexMetaData indexMetaData = metaDataBuilder.get(index);
-                        if (indexMetaData == null) {
-                            throw new IndexNotFoundException(index);
-                        }
+                    for (Index index : openIndices) {
+                        IndexMetaData indexMetaData = metaDataBuilder.getSafe(index);
                         Settings.Builder updates = Settings.builder();
                         Settings.Builder indexSettings = Settings.builder().put(indexMetaData.getSettings());
-                        if (indexScopedSettings.updateDynamicSettings(openSettings, indexSettings, updates, index)) {
+                        if (indexScopedSettings.updateDynamicSettings(openSettings, indexSettings, updates, index.getName())) {
+                            if (preserveExisting) {
+                                indexSettings.put(indexMetaData.getSettings());
+                            }
                             metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(indexSettings));
                         }
                     }
                 }
 
                 if (!closeIndices.isEmpty()) {
-                    for (String index : closeIndices) {
-                        IndexMetaData indexMetaData = metaDataBuilder.get(index);
-                        if (indexMetaData == null) {
-                            throw new IndexNotFoundException(index);
-                        }
+                    for (Index index : closeIndices) {
+                        IndexMetaData indexMetaData = metaDataBuilder.getSafe(index);
                         Settings.Builder updates = Settings.builder();
                         Settings.Builder indexSettings = Settings.builder().put(indexMetaData.getSettings());
-                        if (indexScopedSettings.updateSettings(closedSettings, indexSettings, updates, index)) {
+                        if (indexScopedSettings.updateSettings(closedSettings, indexSettings, updates, index.getName())) {
+                            if (preserveExisting) {
+                                indexSettings.put(indexMetaData.getSettings());
+                            }
                             metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(indexSettings));
                         }
                     }
@@ -265,11 +268,11 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
                 // now, reroute in case things change that require it (like number of replicas)
                 RoutingAllocation.Result routingResult = allocationService.reroute(updatedState, "settings update");
                 updatedState = ClusterState.builder(updatedState).routingResult(routingResult).build();
-                for (String index : openIndices) {
-                    indexScopedSettings.dryRun(updatedState.metaData().index(index).getSettings());
+                for (Index index : openIndices) {
+                    indexScopedSettings.dryRun(updatedState.metaData().getIndexSafe(index).getSettings());
                 }
-                for (String index : closeIndices) {
-                    indexScopedSettings.dryRun(updatedState.metaData().index(index).getSettings());
+                for (Index index : closeIndices) {
+                    indexScopedSettings.dryRun(updatedState.metaData().getIndexSafe(index).getSettings());
                 }
                 return updatedState;
             }
diff --git a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java
index 83f603d2890..ccd30a99e9c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java
@@ -19,24 +19,41 @@
 
 package org.elasticsearch.cluster.node;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
 
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.concurrent.CopyOnWriteArrayList;
 
 /**
  */
 public class DiscoveryNodeService extends AbstractComponent {
 
+    public static final Setting<Long> NODE_ID_SEED_SETTING =
+            // don't use node.id.seed so it won't be seen as an attribute
+            Setting.longSetting("node_id.seed", 0L, Long.MIN_VALUE, Property.NodeScope);
     private final List<CustomAttributesProvider> customAttributesProviders = new CopyOnWriteArrayList<>();
+    private final Version version;
 
     @Inject
-    public DiscoveryNodeService(Settings settings) {
+    public DiscoveryNodeService(Settings settings, Version version) {
         super(settings);
+        this.version = version;
+    }
+
+    public static String generateNodeId(Settings settings) {
+        Random random = Randomness.get(settings, NODE_ID_SEED_SETTING);
+        return Strings.randomBase64UUID(random);
     }
 
     public DiscoveryNodeService addCustomAttributeProvider(CustomAttributesProvider customAttributesProvider) {
@@ -44,7 +61,7 @@ public class DiscoveryNodeService extends AbstractComponent {
         return this;
     }
 
-    public Map<String, String> buildAttributes() {
+    public DiscoveryNode buildLocalNode(TransportAddress publishAddress) {
         Map<String, String> attributes = new HashMap<>(settings.getByPrefix("node.").getAsMap());
         attributes.remove("name"); // name is extracted in other places
         if (attributes.containsKey("client")) {
@@ -76,10 +93,11 @@ public class DiscoveryNodeService extends AbstractComponent {
             }
         }
 
-        return attributes;
+        final String nodeId = generateNodeId(settings);
+        return new DiscoveryNode(settings.get("node.name"), nodeId, publishAddress, attributes, version);
     }
 
-    public static interface CustomAttributesProvider {
+    public interface CustomAttributesProvider {
 
         Map<String, String> buildAttributes();
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
index 160ccbf06b3..c32d9de363d 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
@@ -313,7 +313,7 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
 
     @Override
     public IndexRoutingTable readFrom(StreamInput in) throws IOException {
-        Index index = Index.readIndex(in);
+        Index index = new Index(in);
         Builder builder = new Builder(index);
 
         int size = in.readVInt();
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
index bda0a24c9a4..e64f8f5d77c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
@@ -584,7 +584,7 @@ public class IndexShardRoutingTable implements Iterable<ShardRouting> {
         }
 
         public static IndexShardRoutingTable readFrom(StreamInput in) throws IOException {
-            Index index = Index.readIndex(in);
+            Index index = new Index(in);
             return readFromThin(in, index);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
index 4f2f9d06097..a6ef564904c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
@@ -597,6 +597,13 @@ public class RoutingNodes implements Iterable<RoutingNode> {
 
     }
 
+    /**
+     * Returns the number of routing nodes
+     */
+    public int size() {
+        return nodesToShards.size();
+    }
+
     public static final class UnassignedShards implements Iterable<ShardRouting>  {
 
         private final RoutingNodes nodes;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
index c683f0200dc..90565a6569d 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
@@ -20,12 +20,12 @@
 package org.elasticsearch.cluster.routing;
 
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java
index a34405c09e0..c27e0a9beb1 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java
@@ -586,10 +586,6 @@ public class RoutingTable implements Iterable<IndexRoutingTable>, Diffable<Routi
             if (indicesRouting == null) {
                 throw new IllegalStateException("once build is called the builder cannot be reused");
             }
-            // normalize the versions right before we build it...
-            for (ObjectCursor<IndexRoutingTable> indexRoutingTable : indicesRouting.values()) {
-                indicesRouting.put(indexRoutingTable.value.getIndex().getName(), indexRoutingTable.value);
-            }
             RoutingTable table = new RoutingTable(version, indicesRouting.build());
             indicesRouting = null;
             return table;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
index cfa33e4f225..269a238e710 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
@@ -266,7 +266,7 @@ public final class ShardRouting implements Streamable, ToXContent {
             return false;
         }
 
-        if (indexMetaData.activeAllocationIds(id()).isEmpty() && indexMetaData.getCreationVersion().onOrAfter(Version.V_5_0_0)) {
+        if (indexMetaData.activeAllocationIds(id()).isEmpty() && indexMetaData.getCreationVersion().onOrAfter(Version.V_5_0_0_alpha1)) {
             // when no shards with this id have ever been active for this index
             return false;
         }
@@ -328,7 +328,7 @@ public final class ShardRouting implements Streamable, ToXContent {
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
-        readFrom(in, Index.readIndex(in), in.readVInt());
+        readFrom(in, new Index(in), in.readVInt());
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
index 714c1e4913a..be7d90a1fef 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
@@ -29,6 +29,7 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -44,7 +45,9 @@ public class UnassignedInfo implements ToXContent, Writeable<UnassignedInfo> {
     public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("dateOptionalTime");
     private static final TimeValue DEFAULT_DELAYED_NODE_LEFT_TIMEOUT = TimeValue.timeValueMinutes(1);
 
-    public static final Setting<TimeValue> INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING = Setting.timeSetting("index.unassigned.node_left.delayed_timeout", DEFAULT_DELAYED_NODE_LEFT_TIMEOUT, true, Setting.Scope.INDEX);
+    public static final Setting<TimeValue> INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING =
+        Setting.timeSetting("index.unassigned.node_left.delayed_timeout", DEFAULT_DELAYED_NODE_LEFT_TIMEOUT, Property.Dynamic,
+            Property.IndexScope);
 
     /**
      * Reason why the shard is in unassigned state.
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
index eeeb6e3389c..da0fea69c68 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.cluster.routing.allocation;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
-import org.apache.lucene.util.ArrayUtil;
 import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
@@ -36,13 +35,14 @@ import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
-import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocators;
+import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
-import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.gateway.GatewayAllocator;
+import org.elasticsearch.index.shard.ShardId;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -63,14 +63,17 @@ import java.util.stream.Collectors;
 public class AllocationService extends AbstractComponent {
 
     private final AllocationDeciders allocationDeciders;
+    private final GatewayAllocator gatewayAllocator;
+    private final ShardsAllocator shardsAllocator;
     private final ClusterInfoService clusterInfoService;
-    private final ShardsAllocators shardsAllocators;
 
     @Inject
-    public AllocationService(Settings settings, AllocationDeciders allocationDeciders, ShardsAllocators shardsAllocators, ClusterInfoService clusterInfoService) {
+    public AllocationService(Settings settings, AllocationDeciders allocationDeciders, GatewayAllocator gatewayAllocator,
+                             ShardsAllocator shardsAllocator, ClusterInfoService clusterInfoService) {
         super(settings);
         this.allocationDeciders = allocationDeciders;
-        this.shardsAllocators = shardsAllocators;
+        this.gatewayAllocator = gatewayAllocator;
+        this.shardsAllocator = shardsAllocator;
         this.clusterInfoService = clusterInfoService;
     }
 
@@ -92,11 +95,11 @@ public class AllocationService extends AbstractComponent {
         if (!changed) {
             return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData());
         }
-        shardsAllocators.applyStartedShards(allocation);
+        gatewayAllocator.applyStartedShards(allocation);
         if (withReroute) {
             reroute(allocation);
         }
-        final RoutingAllocation.Result result = buildChangedResult(clusterState.metaData(), routingNodes);
+        final RoutingAllocation.Result result = buildChangedResult(clusterState.metaData(), clusterState.routingTable(), routingNodes);
 
         String startedShardsAsString = firstListElementsToCommaDelimitedString(startedShards, s -> s.shardId().toString());
         logClusterHealthStateChange(
@@ -105,37 +108,44 @@ public class AllocationService extends AbstractComponent {
                 "shards started [" + startedShardsAsString + "] ..."
         );
         return result;
-    }
-
-
-    protected RoutingAllocation.Result buildChangedResult(MetaData metaData, RoutingNodes routingNodes) {
-        return buildChangedResult(metaData, routingNodes, new RoutingExplanations());
 
     }
-    protected RoutingAllocation.Result buildChangedResult(MetaData metaData, RoutingNodes routingNodes, RoutingExplanations explanations) {
-        final RoutingTable routingTable = new RoutingTable.Builder().updateNodes(routingNodes).build();
-        MetaData newMetaData = updateMetaDataWithRoutingTable(metaData,routingTable);
-        return new RoutingAllocation.Result(true, routingTable.validateRaiseException(newMetaData), newMetaData, explanations);
+
+    protected RoutingAllocation.Result buildChangedResult(MetaData oldMetaData, RoutingTable oldRoutingTable, RoutingNodes newRoutingNodes) {
+        return buildChangedResult(oldMetaData, oldRoutingTable, newRoutingNodes, new RoutingExplanations());
+
+    }
+
+    protected RoutingAllocation.Result buildChangedResult(MetaData oldMetaData, RoutingTable oldRoutingTable, RoutingNodes newRoutingNodes,
+                                                          RoutingExplanations explanations) {
+        final RoutingTable newRoutingTable = new RoutingTable.Builder().updateNodes(newRoutingNodes).build();
+        MetaData newMetaData = updateMetaDataWithRoutingTable(oldMetaData, oldRoutingTable, newRoutingTable);
+        return new RoutingAllocation.Result(true, newRoutingTable.validateRaiseException(newMetaData), newMetaData, explanations);
     }
 
     /**
-     * Updates the current {@link MetaData} based on the newly created {@link RoutingTable}.
+     * Updates the current {@link MetaData} based on the newly created {@link RoutingTable}. Specifically
+     * we update {@link IndexMetaData#getActiveAllocationIds()} and {@link IndexMetaData#primaryTerm(int)} based on
+     * the changes made during this allocation.
      *
-     * @param currentMetaData {@link MetaData} object from before the routing table was changed.
+     * @param oldMetaData     {@link MetaData} object from before the routing table was changed.
+     * @param oldRoutingTable {@link RoutingTable} from before the  change.
      * @param newRoutingTable new {@link RoutingTable} created by the allocation change
      * @return adapted {@link MetaData}, potentially the original one if no change was needed.
      */
-    static MetaData updateMetaDataWithRoutingTable(MetaData currentMetaData, RoutingTable newRoutingTable) {
-        // make sure index meta data and routing tables are in sync w.r.t active allocation ids
+    static MetaData updateMetaDataWithRoutingTable(MetaData oldMetaData, RoutingTable oldRoutingTable, RoutingTable newRoutingTable) {
         MetaData.Builder metaDataBuilder = null;
-        for (IndexRoutingTable indexRoutingTable : newRoutingTable) {
-            final IndexMetaData indexMetaData = currentMetaData.index(indexRoutingTable.getIndex());
-            if (indexMetaData == null) {
-                throw new IllegalStateException("no metadata found for index " + indexRoutingTable.getIndex().getName());
+        for (IndexRoutingTable newIndexTable : newRoutingTable) {
+            final IndexMetaData oldIndexMetaData = oldMetaData.index(newIndexTable.getIndex());
+            if (oldIndexMetaData == null) {
+                throw new IllegalStateException("no metadata found for index " + newIndexTable.getIndex().getName());
             }
             IndexMetaData.Builder indexMetaDataBuilder = null;
-            for (IndexShardRoutingTable shardRoutings : indexRoutingTable) {
-                Set<String> activeAllocationIds = shardRoutings.activeShards().stream()
+            for (IndexShardRoutingTable newShardTable : newIndexTable) {
+                final ShardId shardId = newShardTable.shardId();
+
+                // update activeAllocationIds
+                Set<String> activeAllocationIds = newShardTable.activeShards().stream()
                         .map(ShardRouting::allocationId)
                         .filter(Objects::nonNull)
                         .map(AllocationId::getId)
@@ -143,19 +153,44 @@ public class AllocationService extends AbstractComponent {
                 // only update active allocation ids if there is an active shard
                 if (activeAllocationIds.isEmpty() == false) {
                     // get currently stored allocation ids
-                    Set<String> storedAllocationIds = indexMetaData.activeAllocationIds(shardRoutings.shardId().id());
+                    Set<String> storedAllocationIds = oldIndexMetaData.activeAllocationIds(shardId.id());
                     if (activeAllocationIds.equals(storedAllocationIds) == false) {
                         if (indexMetaDataBuilder == null) {
-                            indexMetaDataBuilder = IndexMetaData.builder(indexMetaData);
+                            indexMetaDataBuilder = IndexMetaData.builder(oldIndexMetaData);
                         }
-
-                        indexMetaDataBuilder.putActiveAllocationIds(shardRoutings.shardId().id(), activeAllocationIds);
+                        indexMetaDataBuilder.putActiveAllocationIds(shardId.id(), activeAllocationIds);
                     }
                 }
+
+                // update primary terms
+                final ShardRouting newPrimary = newShardTable.primaryShard();
+                if (newPrimary == null) {
+                    throw new IllegalStateException("missing primary shard for " + newShardTable.shardId());
+                }
+                final ShardRouting oldPrimary = oldRoutingTable.shardRoutingTable(shardId).primaryShard();
+                if (oldPrimary == null) {
+                    throw new IllegalStateException("missing primary shard for " + newShardTable.shardId());
+                }
+                // we update the primary term on initial assignment or when a replica is promoted. Most notably we do *not*
+                // update them when a primary relocates
+                if (newPrimary.unassigned() ||
+                        newPrimary.isSameAllocation(oldPrimary) ||
+                        // we do not use newPrimary.isTargetRelocationOf(oldPrimary) because that one enforces newPrimary to
+                        // be initializing. However, when the target shard is activated, we still want the primary term to staty
+                        // the same
+                        (oldPrimary.relocating() && newPrimary.isSameAllocation(oldPrimary.buildTargetRelocatingShard()))) {
+                    // do nothing
+                } else {
+                    // incrementing the primary term
+                    if (indexMetaDataBuilder == null) {
+                        indexMetaDataBuilder = IndexMetaData.builder(oldIndexMetaData);
+                    }
+                    indexMetaDataBuilder.primaryTerm(shardId.id(), oldIndexMetaData.primaryTerm(shardId.id()) + 1);
+                }
             }
             if (indexMetaDataBuilder != null) {
                 if (metaDataBuilder == null) {
-                    metaDataBuilder = MetaData.builder(currentMetaData);
+                    metaDataBuilder = MetaData.builder(oldMetaData);
                 }
                 metaDataBuilder.put(indexMetaDataBuilder);
             }
@@ -163,7 +198,7 @@ public class AllocationService extends AbstractComponent {
         if (metaDataBuilder != null) {
             return metaDataBuilder.build();
         } else {
-            return currentMetaData;
+            return oldMetaData;
         }
     }
 
@@ -192,9 +227,9 @@ public class AllocationService extends AbstractComponent {
         if (!changed) {
             return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData());
         }
-        shardsAllocators.applyFailedShards(allocation);
+        gatewayAllocator.applyFailedShards(allocation);
         reroute(allocation);
-        final RoutingAllocation.Result result = buildChangedResult(clusterState.metaData(), routingNodes);
+        final RoutingAllocation.Result result = buildChangedResult(clusterState.metaData(), clusterState.routingTable(), routingNodes);
         String failedShardsAsString = firstListElementsToCommaDelimitedString(failedShards, s -> s.shard.shardId().toString());
         logClusterHealthStateChange(
                 new ClusterStateHealth(clusterState),
@@ -241,7 +276,7 @@ public class AllocationService extends AbstractComponent {
         // the assumption is that commands will move / act on shards (or fail through exceptions)
         // so, there will always be shard "movements", so no need to check on reroute
         reroute(allocation);
-        RoutingAllocation.Result result = buildChangedResult(clusterState.metaData(), routingNodes, explanations);
+        RoutingAllocation.Result result = buildChangedResult(clusterState.metaData(), clusterState.routingTable(), routingNodes, explanations);
         logClusterHealthStateChange(
                 new ClusterStateHealth(clusterState),
                 new ClusterStateHealth(clusterState.getMetaData(), result.routingTable()),
@@ -250,6 +285,7 @@ public class AllocationService extends AbstractComponent {
         return result;
     }
 
+
     /**
      * Reroutes the routing table based on the live nodes.
      * <p>
@@ -273,7 +309,7 @@ public class AllocationService extends AbstractComponent {
         if (!reroute(allocation)) {
             return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData());
         }
-        RoutingAllocation.Result result = buildChangedResult(clusterState.metaData(), routingNodes);
+        RoutingAllocation.Result result = buildChangedResult(clusterState.metaData(), clusterState.routingTable(), routingNodes);
         logClusterHealthStateChange(
                 new ClusterStateHealth(clusterState),
                 new ClusterStateHealth(clusterState.getMetaData(), result.routingTable()),
@@ -306,14 +342,10 @@ public class AllocationService extends AbstractComponent {
         if (allocation.routingNodes().unassigned().size() > 0) {
             updateLeftDelayOfUnassignedShards(allocation, settings);
 
-            changed |= shardsAllocators.allocateUnassigned(allocation);
+            changed |= gatewayAllocator.allocateUnassigned(allocation);
         }
 
-        // move shards that no longer can be allocated
-        changed |= shardsAllocators.moveShards(allocation);
-
-        // rebalance
-        changed |= shardsAllocators.rebalance(allocation);
+        changed |= shardsAllocator.allocate(allocation);
         assert RoutingNodes.assertShardStats(allocation.routingNodes());
         return changed;
     }
@@ -322,7 +354,7 @@ public class AllocationService extends AbstractComponent {
     public static void updateLeftDelayOfUnassignedShards(RoutingAllocation allocation, Settings settings) {
         for (ShardRouting shardRouting : allocation.routingNodes().unassigned()) {
             final MetaData metaData = allocation.metaData();
-            final IndexMetaData indexMetaData = metaData.index(shardRouting.index());
+            final IndexMetaData indexMetaData = metaData.getIndexSafe(shardRouting.index());
             shardRouting.unassignedInfo().updateDelay(allocation.getCurrentNanoTime(), settings, indexMetaData.getSettings());
         }
     }
@@ -342,7 +374,6 @@ public class AllocationService extends AbstractComponent {
                 changed |= failReplicasForUnassignedPrimary(allocation, shardEntry);
                 ShardRouting candidate = allocation.routingNodes().activeReplica(shardEntry);
                 if (candidate != null) {
-                    IndexMetaData index = allocation.metaData().index(candidate.index());
                     routingNodes.swapPrimaryFlag(shardEntry, candidate);
                     if (candidate.relocatingNodeId() != null) {
                         changed = true;
@@ -357,6 +388,7 @@ public class AllocationService extends AbstractComponent {
                             }
                         }
                     }
+                    IndexMetaData index = allocation.metaData().getIndexSafe(candidate.index());
                     if (IndexMetaData.isIndexUsingShadowReplicas(index.getSettings())) {
                         routingNodes.reinitShadowPrimary(candidate);
                         changed = true;
@@ -414,8 +446,8 @@ public class AllocationService extends AbstractComponent {
         boolean changed = false;
         for (ShardRouting routing : replicas) {
             changed |= applyFailedShard(allocation, routing, false,
-                new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, "primary failed while replica initializing",
-                    null, allocation.getCurrentNanoTime(), System.currentTimeMillis()));
+                    new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, "primary failed while replica initializing",
+                            null, allocation.getCurrentNanoTime(), System.currentTimeMillis()));
         }
         return changed;
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
index 4e6ba0fb5ad..536806c0830 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
@@ -44,7 +44,7 @@ import static java.util.Collections.unmodifiableSet;
 public class RoutingAllocation {
 
     /**
-     * this class is used to describe results of a {@link RoutingAllocation}  
+     * this class is used to describe results of a {@link RoutingAllocation}
      */
     public static class Result {
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
index 0c40b26ca67..8102f206799 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.cluster.routing.allocation.allocator;
 
-import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.IntroSorter;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -28,9 +27,7 @@ import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
-import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
@@ -39,21 +36,18 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.gateway.PriorityComparator;
 
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.IdentityHashMap;
 import java.util.Iterator;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.function.Predicate;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
 
@@ -74,9 +68,13 @@ import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
  */
 public class BalancedShardsAllocator extends AbstractComponent implements ShardsAllocator {
 
-    public static final Setting<Float> INDEX_BALANCE_FACTOR_SETTING = Setting.floatSetting("cluster.routing.allocation.balance.index", 0.55f, true, Setting.Scope.CLUSTER);
-    public static final Setting<Float> SHARD_BALANCE_FACTOR_SETTING = Setting.floatSetting("cluster.routing.allocation.balance.shard", 0.45f, true, Setting.Scope.CLUSTER);
-    public static final Setting<Float> THRESHOLD_SETTING = Setting.floatSetting("cluster.routing.allocation.balance.threshold", 1.0f, 0.0f, true, Setting.Scope.CLUSTER);
+    public static final Setting<Float> INDEX_BALANCE_FACTOR_SETTING =
+        Setting.floatSetting("cluster.routing.allocation.balance.index", 0.55f, Property.Dynamic, Property.NodeScope);
+    public static final Setting<Float> SHARD_BALANCE_FACTOR_SETTING =
+        Setting.floatSetting("cluster.routing.allocation.balance.shard", 0.45f, Property.Dynamic, Property.NodeScope);
+    public static final Setting<Float> THRESHOLD_SETTING =
+        Setting.floatSetting("cluster.routing.allocation.balance.threshold", 1.0f, 0.0f,
+            Property.Dynamic, Property.NodeScope);
 
     private volatile WeightFunction weightFunction;
     private volatile float threshold;
@@ -103,27 +101,16 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
     }
 
     @Override
-    public void applyStartedShards(StartedRerouteAllocation allocation) { /* ONLY FOR GATEWAYS */ }
-
-    @Override
-    public void applyFailedShards(FailedRerouteAllocation allocation) { /* ONLY FOR GATEWAYS */ }
-
-    @Override
-    public boolean allocateUnassigned(RoutingAllocation allocation) {
+    public boolean allocate(RoutingAllocation allocation) {
+        if (allocation.routingNodes().size() == 0) {
+            /* with no nodes this is pointless */
+            return false;
+        }
         final Balancer balancer = new Balancer(logger, allocation, weightFunction, threshold);
-        return balancer.allocateUnassigned();
-    }
-
-    @Override
-    public boolean rebalance(RoutingAllocation allocation) {
-        final Balancer balancer = new Balancer(logger, allocation, weightFunction, threshold);
-        return balancer.balance();
-    }
-
-    @Override
-    public boolean moveShards(RoutingAllocation allocation) {
-        final Balancer balancer = new Balancer(logger, allocation, weightFunction, threshold);
-        return balancer.moveShards();
+        boolean changed = balancer.allocateUnassigned();
+        changed |= balancer.moveShards();
+        changed |= balancer.balance();
+        return changed;
     }
 
     /**
@@ -203,8 +190,8 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
         }
 
         private float weight(Balancer balancer, ModelNode node, String index, int numAdditionalShards) {
-            final float weightShard = (node.numShards() + numAdditionalShards - balancer.avgShardsPerNode());
-            final float weightIndex = (node.numShards(index) + numAdditionalShards - balancer.avgShardsPerNode(index));
+            final float weightShard = node.numShards() + numAdditionalShards - balancer.avgShardsPerNode();
+            final float weightIndex = node.numShards(index) + numAdditionalShards - balancer.avgShardsPerNode(index);
             return theta0 * weightShard + theta1 * weightIndex;
         }
 
@@ -216,7 +203,6 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
     public static class Balancer {
         private final ESLogger logger;
         private final Map<String, ModelNode> nodes = new HashMap<>();
-        private final HashSet<String> indices = new HashSet<>();
         private final RoutingAllocation allocation;
         private final RoutingNodes routingNodes;
         private final WeightFunction weight;
@@ -225,19 +211,15 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
         private final MetaData metaData;
         private final float avgShardsPerNode;
 
-        private final Predicate<ShardRouting> assignedFilter = shard -> shard.assignedToNode();
-
         public Balancer(ESLogger logger, RoutingAllocation allocation, WeightFunction weight, float threshold) {
             this.logger = logger;
             this.allocation = allocation;
             this.weight = weight;
             this.threshold = threshold;
             this.routingNodes = allocation.routingNodes();
-            for (RoutingNode node : routingNodes) {
-                nodes.put(node.nodeId(), new ModelNode(node.nodeId()));
-            }
             metaData = routingNodes.metaData();
-            avgShardsPerNode = ((float) metaData.totalNumberOfShards()) / nodes.size();
+            avgShardsPerNode = ((float) metaData.getTotalNumberOfShards()) / routingNodes.size();
+            buildModelFromAssigned();
         }
 
         /**
@@ -271,17 +253,6 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
             return new NodeSorter(nodesArray(), weight, this);
         }
 
-        private boolean initialize(RoutingNodes routing, RoutingNodes.UnassignedShards unassigned) {
-            if (logger.isTraceEnabled()) {
-                logger.trace("Start distributing Shards");
-            }
-            for (ObjectCursor<String> index : allocation.routingTable().indicesRouting().keys()) {
-                indices.add(index.value);
-            }
-            buildModelFromAssigned(routing.shards(assignedFilter));
-            return allocateUnassigned(unassigned);
-        }
-
         private static float absDelta(float lower, float higher) {
             assert higher >= lower : higher + " lt " + lower +" but was expected to be gte";
             return Math.abs(higher - lower);
@@ -295,12 +266,36 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
         }
 
         /**
-         * Allocates all possible unassigned shards
+         * Balances the nodes on the cluster model according to the weight function.
+         * The actual balancing is delegated to {@link #balanceByWeights()}
+         *
          * @return <code>true</code> if the current configuration has been
          *         changed, otherwise <code>false</code>
          */
-        final boolean allocateUnassigned() {
-            return balance(true);
+        private boolean balance() {
+            if (logger.isTraceEnabled()) {
+                logger.trace("Start balancing cluster");
+            }
+            if (allocation.hasPendingAsyncFetch()) {
+                /*
+                 * see https://github.com/elastic/elasticsearch/issues/14387
+                 * if we allow rebalance operations while we are still fetching shard store data
+                 * we might end up with unnecessary rebalance operations which can be super confusion/frustrating
+                 * since once the fetches come back we might just move all the shards back again.
+                 * Therefore we only do a rebalance if we have fetched all information.
+                 */
+                logger.debug("skipping rebalance due to in-flight shard/store fetches");
+                return false;
+            }
+            if (allocation.deciders().canRebalance(allocation).type() != Type.YES) {
+                logger.trace("skipping rebalance as it is disabled");
+                return false;
+            }
+            if (nodes.size() < 2) { /* skip if we only have one node */
+                logger.trace("skipping rebalance as single node only");
+                return false;
+            }
+            return balanceByWeights();
         }
 
         /**
@@ -317,120 +312,100 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
          * @return <code>true</code> if the current configuration has been
          *         changed, otherwise <code>false</code>
          */
-        public boolean balance() {
-            return balance(false);
-        }
+        private boolean balanceByWeights() {
+            boolean changed = false;
+            final NodeSorter sorter = newNodeSorter();
+            final AllocationDeciders deciders = allocation.deciders();
+            final ModelNode[] modelNodes = sorter.modelNodes;
+            final float[] weights = sorter.weights;
+            for (String index : buildWeightOrderedIndices(sorter)) {
+                IndexMetaData indexMetaData = metaData.index(index);
 
-        private boolean balance(boolean onlyAssign) {
-            if (this.nodes.isEmpty()) {
-                /* with no nodes this is pointless */
-                return false;
-            }
-            if (logger.isTraceEnabled()) {
-                if (onlyAssign) {
-                    logger.trace("Start balancing cluster");
-                } else {
-                    logger.trace("Start assigning unassigned shards");
+                // find nodes that have a shard of this index or where shards of this index are allowed to stay
+                // move these nodes to the front of modelNodes so that we can only balance based on these nodes
+                int relevantNodes = 0;
+                for (int i = 0; i < modelNodes.length; i++) {
+                    ModelNode modelNode = modelNodes[i];
+                    if (modelNode.getIndex(index) != null
+                        || deciders.canAllocate(indexMetaData, modelNode.getRoutingNode(), allocation).type() != Type.NO) {
+                        // swap nodes at position i and relevantNodes
+                        modelNodes[i] = modelNodes[relevantNodes];
+                        modelNodes[relevantNodes] = modelNode;
+                        relevantNodes++;
+                    }
                 }
-            }
-            final RoutingNodes.UnassignedShards unassigned = routingNodes.unassigned();
-            boolean changed = initialize(routingNodes, unassigned);
-            if (onlyAssign == false && changed == false && allocation.deciders().canRebalance(allocation).type() == Type.YES) {
-                NodeSorter sorter = newNodeSorter();
-                if (nodes.size() > 1) { /* skip if we only have one node */
-                    AllocationDeciders deciders = allocation.deciders();
-                    final ModelNode[] modelNodes = sorter.modelNodes;
-                    final float[] weights = sorter.weights;
-                    for (String index : buildWeightOrderedIndices(sorter)) {
-                        IndexMetaData indexMetaData = metaData.index(index);
 
-                        // find nodes that have a shard of this index or where shards of this index are allowed to stay
-                        // move these nodes to the front of modelNodes so that we can only balance based on these nodes
-                        int relevantNodes = 0;
-                        for (int i = 0; i < modelNodes.length; i++) {
-                            ModelNode modelNode = modelNodes[i];
-                            if (modelNode.getIndex(index) != null
-                                || deciders.canAllocate(indexMetaData, modelNode.getRoutingNode(routingNodes), allocation).type() != Type.NO) {
-                                // swap nodes at position i and relevantNodes
-                                modelNodes[i] = modelNodes[relevantNodes];
-                                modelNodes[relevantNodes] = modelNode;
-                                relevantNodes++;
+                if (relevantNodes < 2) {
+                    continue;
+                }
+
+                sorter.reset(index, 0, relevantNodes);
+                int lowIdx = 0;
+                int highIdx = relevantNodes - 1;
+                while (true) {
+                    final ModelNode minNode = modelNodes[lowIdx];
+                    final ModelNode maxNode = modelNodes[highIdx];
+                    advance_range:
+                    if (maxNode.numShards(index) > 0) {
+                        final float delta = absDelta(weights[lowIdx], weights[highIdx]);
+                        if (lessThan(delta, threshold)) {
+                            if (lowIdx > 0 && highIdx-1 > 0 // is there a chance for a higher delta?
+                                && (absDelta(weights[0], weights[highIdx-1]) > threshold) // check if we need to break at all
+                                ) {
+                                /* This is a special case if allocations from the "heaviest" to the "lighter" nodes is not possible
+                                 * due to some allocation decider restrictions like zone awareness. if one zone has for instance
+                                 * less nodes than another zone. so one zone is horribly overloaded from a balanced perspective but we
+                                 * can't move to the "lighter" shards since otherwise the zone would go over capacity.
+                                 *
+                                 * This break jumps straight to the condition below were we start moving from the high index towards
+                                 * the low index to shrink the window we are considering for balance from the other direction.
+                                 * (check shrinking the window from MAX to MIN)
+                                 * See #3580
+                                 */
+                                break advance_range;
                             }
+                            if (logger.isTraceEnabled()) {
+                                logger.trace("Stop balancing index [{}]  min_node [{}] weight: [{}]  max_node [{}] weight: [{}]  delta: [{}]",
+                                        index, maxNode.getNodeId(), weights[highIdx], minNode.getNodeId(), weights[lowIdx], delta);
+                            }
+                            break;
                         }
-
-                        if (relevantNodes < 2) {
+                        if (logger.isTraceEnabled()) {
+                            logger.trace("Balancing from node [{}] weight: [{}] to node [{}] weight: [{}]  delta: [{}]",
+                                    maxNode.getNodeId(), weights[highIdx], minNode.getNodeId(), weights[lowIdx], delta);
+                        }
+                        /* pass the delta to the replication function to prevent relocations that only swap the weights of the two nodes.
+                         * a relocation must bring us closer to the balance if we only achieve the same delta the relocation is useless */
+                        if (tryRelocateShard(minNode, maxNode, index, delta)) {
+                            /*
+                             * TODO we could be a bit smarter here, we don't need to fully sort necessarily
+                             * we could just find the place to insert linearly but the win might be minor
+                             * compared to the added complexity
+                             */
+                            weights[lowIdx] = sorter.weight(modelNodes[lowIdx]);
+                            weights[highIdx] = sorter.weight(modelNodes[highIdx]);
+                            sorter.sort(0, relevantNodes);
+                            lowIdx = 0;
+                            highIdx = relevantNodes - 1;
+                            changed = true;
                             continue;
                         }
-
-                        sorter.reset(index, 0, relevantNodes);
-                        int lowIdx = 0;
-                        int highIdx = relevantNodes - 1;
-                        while (true) {
-                            final ModelNode minNode = modelNodes[lowIdx];
-                            final ModelNode maxNode = modelNodes[highIdx];
-                            advance_range:
-                            if (maxNode.numShards(index) > 0) {
-                                final float delta = absDelta(weights[lowIdx], weights[highIdx]);
-                                if (lessThan(delta, threshold)) {
-                                    if (lowIdx > 0 && highIdx-1 > 0 // is there a chance for a higher delta?
-                                        && (absDelta(weights[0], weights[highIdx-1]) > threshold) // check if we need to break at all
-                                        ) {
-                                        /* This is a special case if allocations from the "heaviest" to the "lighter" nodes is not possible
-                                         * due to some allocation decider restrictions like zone awareness. if one zone has for instance
-                                         * less nodes than another zone. so one zone is horribly overloaded from a balanced perspective but we
-                                         * can't move to the "lighter" shards since otherwise the zone would go over capacity.
-                                         *
-                                         * This break jumps straight to the condition below were we start moving from the high index towards
-                                         * the low index to shrink the window we are considering for balance from the other direction.
-                                         * (check shrinking the window from MAX to MIN)
-                                         * See #3580
-                                         */
-                                        break advance_range;
-                                    }
-                                    if (logger.isTraceEnabled()) {
-                                        logger.trace("Stop balancing index [{}]  min_node [{}] weight: [{}]  max_node [{}] weight: [{}]  delta: [{}]",
-                                                index, maxNode.getNodeId(), weights[highIdx], minNode.getNodeId(), weights[lowIdx], delta);
-                                    }
-                                    break;
-                                }
-                                if (logger.isTraceEnabled()) {
-                                    logger.trace("Balancing from node [{}] weight: [{}] to node [{}] weight: [{}]  delta: [{}]",
-                                            maxNode.getNodeId(), weights[highIdx], minNode.getNodeId(), weights[lowIdx], delta);
-                                }
-                                /* pass the delta to the replication function to prevent relocations that only swap the weights of the two nodes.
-                                 * a relocation must bring us closer to the balance if we only achieve the same delta the relocation is useless */
-                                if (tryRelocateShard(minNode, maxNode, index, delta)) {
-                                    /*
-                                     * TODO we could be a bit smarter here, we don't need to fully sort necessarily
-                                     * we could just find the place to insert linearly but the win might be minor
-                                     * compared to the added complexity
-                                     */
-                                    weights[lowIdx] = sorter.weight(modelNodes[lowIdx]);
-                                    weights[highIdx] = sorter.weight(modelNodes[highIdx]);
-                                    sorter.sort(0, relevantNodes);
-                                    lowIdx = 0;
-                                    highIdx = relevantNodes - 1;
-                                    changed = true;
-                                    continue;
-                                }
-                            }
-                            if (lowIdx < highIdx - 1) {
-                                /* Shrinking the window from MIN to MAX
-                                 * we can't move from any shard from the min node lets move on to the next node
-                                 * and see if the threshold still holds. We either don't have any shard of this
-                                 * index on this node of allocation deciders prevent any relocation.*/
-                                lowIdx++;
-                            } else if (lowIdx > 0) {
-                                /* Shrinking the window from MAX to MIN
-                                 * now we go max to min since obviously we can't move anything to the max node
-                                 * lets pick the next highest */
-                                lowIdx = 0;
-                                highIdx--;
-                            } else {
-                                /* we are done here, we either can't relocate anymore or we are balanced */
-                                break;
-                            }
-                        }
+                    }
+                    if (lowIdx < highIdx - 1) {
+                        /* Shrinking the window from MIN to MAX
+                         * we can't move from any shard from the min node lets move on to the next node
+                         * and see if the threshold still holds. We either don't have any shard of this
+                         * index on this node of allocation deciders prevent any relocation.*/
+                        lowIdx++;
+                    } else if (lowIdx > 0) {
+                        /* Shrinking the window from MAX to MIN
+                         * now we go max to min since obviously we can't move anything to the max node
+                         * lets pick the next highest */
+                        lowIdx = 0;
+                        highIdx--;
+                    } else {
+                        /* we are done here, we either can't relocate anymore or we are balanced */
+                        break;
                     }
                 }
             }
@@ -451,7 +426,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
          * to the nodes we relocated them from.
          */
         private String[] buildWeightOrderedIndices(NodeSorter sorter) {
-            final String[] indices = this.indices.toArray(new String[this.indices.size()]);
+            final String[] indices = allocation.routingTable().indicesRouting().keys().toArray(String.class);
             final float[] deltas = new float[indices.length];
             for (int i = 0; i < deltas.length; i++) {
                 sorter.reset(indices[i]);
@@ -503,20 +478,16 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
          * @return <code>true</code> if the allocation has changed, otherwise <code>false</code>
          */
         public boolean moveShards() {
-            if (nodes.isEmpty()) {
-                /* with no nodes this is pointless */
-                return false;
-            }
-
-            // Create a copy of the started shards interleaving between nodes, and check if they can remain. In the presence of throttling
+            // Iterate over the started shards interleaving between nodes, and check if they can remain. In the presence of throttling
             // shard movements, the goal of this iteration order is to achieve a fairer movement of shards from the nodes that are
             // offloading the shards.
-            List<ShardRouting> shards = new ArrayList<>();
+            boolean changed = false;
             int index = 0;
             boolean found = true;
+            final NodeSorter sorter = newNodeSorter();
             while (found) {
                 found = false;
-                for (RoutingNode routingNode : routingNodes) {
+                for (RoutingNode routingNode : allocation.routingNodes()) {
                     if (index >= routingNode.size()) {
                         continue;
                     }
@@ -524,64 +495,52 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
                     ShardRouting shardRouting = routingNode.get(index);
                     // we can only move started shards...
                     if (shardRouting.started()) {
-                        shards.add(shardRouting);
+                        final ModelNode sourceNode = nodes.get(shardRouting.currentNodeId());
+                        assert sourceNode != null && sourceNode.containsShard(shardRouting);
+                        Decision decision = allocation.deciders().canRemain(shardRouting, routingNode, allocation);
+                        if (decision.type() == Decision.Type.NO) {
+                            changed |= moveShard(sorter, shardRouting, sourceNode, routingNode);
+                        }
                     }
                 }
                 index++;
             }
-            if (shards.isEmpty()) {
-                return false;
-            }
 
-            final RoutingNodes.UnassignedShards unassigned = routingNodes.unassigned();
-            boolean changed = initialize(routingNodes, unassigned);
-            if (changed == false) {
-                final NodeSorter sorter = newNodeSorter();
-                final ModelNode[] modelNodes = sorter.modelNodes;
-                for (ShardRouting shardRouting : shards) {
-                    final ModelNode sourceNode = nodes.get(shardRouting.currentNodeId());
-                    assert sourceNode != null && sourceNode.containsShard(shardRouting);
-                    final RoutingNode routingNode = sourceNode.getRoutingNode(routingNodes);
-                    Decision decision = allocation.deciders().canRemain(shardRouting, routingNode, allocation);
-                    if (decision.type() == Decision.Type.NO) {
-                        logger.debug("[{}][{}] allocated on [{}], but can no longer be allocated on it, moving...", shardRouting.index(), shardRouting.id(), routingNode.node());
-                        sorter.reset(shardRouting.getIndexName());
-                        /*
-                         * the sorter holds the minimum weight node first for the shards index.
-                         * We now walk through the nodes until we find a node to allocate the shard.
-                         * This is not guaranteed to be balanced after this operation we still try best effort to
-                         * allocate on the minimal eligible node.
-                         */
-                        boolean moved = false;
-                        for (ModelNode currentNode : modelNodes) {
-                            if (currentNode == sourceNode) {
-                                continue;
-                            }
-                            RoutingNode target = currentNode.getRoutingNode(routingNodes);
-                            Decision allocationDecision = allocation.deciders().canAllocate(shardRouting, target, allocation);
-                            Decision rebalanceDecision = allocation.deciders().canRebalance(shardRouting, allocation);
-                            if (allocationDecision.type() == Type.YES && rebalanceDecision.type() == Type.YES) { // TODO maybe we can respect throttling here too?
-                                Decision sourceDecision = sourceNode.removeShard(shardRouting);
-                                ShardRouting targetRelocatingShard = routingNodes.relocate(shardRouting, target.nodeId(), allocation.clusterInfo().getShardSize(shardRouting, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
-                                // re-add (now relocating shard) to source node
-                                sourceNode.addShard(shardRouting, sourceDecision);
-                                Decision targetDecision = new Decision.Multi().add(allocationDecision).add(rebalanceDecision);
-                                currentNode.addShard(targetRelocatingShard, targetDecision);
-                                if (logger.isTraceEnabled()) {
-                                    logger.trace("Moved shard [{}] to node [{}]", shardRouting, routingNode.node());
-                                }
-                                moved = true;
-                                changed = true;
-                                break;
-                            }
-                        }
-                        if (moved == false) {
-                            logger.debug("[{}][{}] can't move", shardRouting.index(), shardRouting.id());
+            return changed;
+        }
+
+        /**
+         * Move started shard to the minimal eligible node with respect to the weight function
+         *
+         * @return <code>true</code> if the shard was moved successfully, otherwise <code>false</code>
+         */
+        private boolean moveShard(NodeSorter sorter, ShardRouting shardRouting, ModelNode sourceNode, RoutingNode routingNode) {
+            logger.debug("[{}][{}] allocated on [{}], but can no longer be allocated on it, moving...", shardRouting.index(), shardRouting.id(), routingNode.node());
+            sorter.reset(shardRouting.getIndexName());
+            /*
+             * the sorter holds the minimum weight node first for the shards index.
+             * We now walk through the nodes until we find a node to allocate the shard.
+             * This is not guaranteed to be balanced after this operation we still try best effort to
+             * allocate on the minimal eligible node.
+             */
+            for (ModelNode currentNode : sorter.modelNodes) {
+                if (currentNode != sourceNode) {
+                    RoutingNode target = currentNode.getRoutingNode();
+                    Decision allocationDecision = allocation.deciders().canAllocate(shardRouting, target, allocation);
+                    Decision rebalanceDecision = allocation.deciders().canRebalance(shardRouting, allocation);
+                    if (allocationDecision.type() == Type.YES && rebalanceDecision.type() == Type.YES) { // TODO maybe we can respect throttling here too?
+                        sourceNode.removeShard(shardRouting);
+                        ShardRouting targetRelocatingShard = routingNodes.relocate(shardRouting, target.nodeId(), allocation.clusterInfo().getShardSize(shardRouting, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
+                        currentNode.addShard(targetRelocatingShard);
+                        if (logger.isTraceEnabled()) {
+                            logger.trace("Moved shard [{}] to node [{}]", shardRouting, routingNode.node());
                         }
+                        return true;
                     }
                 }
             }
-            return changed;
+            logger.debug("[{}][{}] can't move", shardRouting.index(), shardRouting.id());
+            return false;
         }
 
         /**
@@ -593,18 +552,19 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
          * on the target node which we respect during the allocation / balancing
          * process. In short, this method recreates the status-quo in the cluster.
          */
-        private void buildModelFromAssigned(Iterable<ShardRouting> shards) {
-            for (ShardRouting shard : shards) {
-                assert shard.assignedToNode();
-                /* we skip relocating shards here since we expect an initializing shard with the same id coming in */
-                if (shard.state() == RELOCATING) {
-                    continue;
-                }
-                ModelNode node = nodes.get(shard.currentNodeId());
-                assert node != null;
-                node.addShard(shard, Decision.single(Type.YES, "Already allocated on node", node.getNodeId()));
-                if (logger.isTraceEnabled()) {
-                    logger.trace("Assigned shard [{}] to node [{}]", shard, node.getNodeId());
+        private void buildModelFromAssigned() {
+            for (RoutingNode rn : routingNodes) {
+                ModelNode node = new ModelNode(rn);
+                nodes.put(rn.nodeId(), node);
+                for (ShardRouting shard : rn) {
+                    assert rn.nodeId().equals(shard.currentNodeId());
+                    /* we skip relocating shards here since we expect an initializing shard with the same id coming in */
+                    if (shard.state() != RELOCATING) {
+                        node.addShard(shard);
+                        if (logger.isTraceEnabled()) {
+                            logger.trace("Assigned shard [{}] to node [{}]", shard, node.getNodeId());
+                        }
+                    }
                 }
             }
         }
@@ -612,8 +572,11 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
         /**
          * Allocates all given shards on the minimal eligible node for the shards index
          * with respect to the weight function. All given shards must be unassigned.
+         * @return <code>true</code> if the current configuration has been
+         *         changed, otherwise <code>false</code>
          */
-        private boolean allocateUnassigned(RoutingNodes.UnassignedShards unassigned) {
+        private boolean allocateUnassigned() {
+            RoutingNodes.UnassignedShards unassigned = routingNodes.unassigned();
             assert !nodes.isEmpty();
             if (logger.isTraceEnabled()) {
                 logger.trace("Start allocating unassigned shards");
@@ -657,7 +620,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
             int secondaryLength = 0;
             int primaryLength = primary.length;
             ArrayUtil.timSort(primary, comparator);
-            final Set<ModelNode> throttledNodes = Collections.newSetFromMap(new IdentityHashMap<ModelNode, Boolean>());
+            final Set<ModelNode> throttledNodes = Collections.newSetFromMap(new IdentityHashMap<>());
             do {
                 for (int i = 0; i < primaryLength; i++) {
                     ShardRouting shard = primary[i];
@@ -695,7 +658,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
                                  * don't check deciders
                                  */
                                 if (currentWeight <= minWeight) {
-                                    Decision currentDecision = deciders.canAllocate(shard, node.getRoutingNode(routingNodes), allocation);
+                                    Decision currentDecision = deciders.canAllocate(shard, node.getRoutingNode(), allocation);
                                     NOUPDATE:
                                     if (currentDecision.type() == Type.YES || currentDecision.type() == Type.THROTTLE) {
                                         if (currentWeight == minWeight) {
@@ -736,7 +699,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
                     }
                     assert decision != null && minNode != null || decision == null && minNode == null;
                     if (minNode != null) {
-                        minNode.addShard(shard, decision);
+                        minNode.addShard(shard);
                         if (decision.type() == Type.YES) {
                             if (logger.isTraceEnabled()) {
                                 logger.trace("Assigned shard [{}] to [{}]", shard, minNode.getNodeId());
@@ -745,7 +708,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
                             changed = true;
                             continue; // don't add to ignoreUnassigned
                         } else {
-                            final RoutingNode node = minNode.getRoutingNode(routingNodes);
+                            final RoutingNode node = minNode.getRoutingNode();
                             if (deciders.canAllocate(node, allocation).type() != Type.YES) {
                                 if (logger.isTraceEnabled()) {
                                     logger.trace("Can not allocate on node [{}] remove from round decision [{}]", node, decision.type());
@@ -791,10 +754,10 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
                 }
                 ShardRouting candidate = null;
                 final AllocationDeciders deciders = allocation.deciders();
-                for (ShardRouting shard : index.getAllShards()) {
+                for (ShardRouting shard : index) {
                     if (shard.started()) {
                         // skip initializing, unassigned and relocating shards we can't relocate them anyway
-                        Decision allocationDecision = deciders.canAllocate(shard, minNode.getRoutingNode(routingNodes), allocation);
+                        Decision allocationDecision = deciders.canAllocate(shard, minNode.getRoutingNode(), allocation);
                         Decision rebalanceDecision = deciders.canRebalance(shard, allocation);
                         if (((allocationDecision.type() == Type.YES) || (allocationDecision.type() == Type.THROTTLE))
                                 && ((rebalanceDecision.type() == Type.YES) || (rebalanceDecision.type() == Type.THROTTLE))) {
@@ -815,24 +778,17 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
                 }
 
                 if (candidate != null) {
-
                     /* allocate on the model even if not throttled */
                     maxNode.removeShard(candidate);
-                    minNode.addShard(candidate, decision);
+                    minNode.addShard(candidate);
                     if (decision.type() == Type.YES) { /* only allocate on the cluster if we are not throttled */
                         if (logger.isTraceEnabled()) {
                             logger.trace("Relocate shard [{}] from node [{}] to node [{}]", candidate, maxNode.getNodeId(),
                                     minNode.getNodeId());
                         }
-                        /* now allocate on the cluster - if we are started we need to relocate the shard */
-                        if (candidate.started()) {
-                            routingNodes.relocate(candidate, minNode.getNodeId(), allocation.clusterInfo().getShardSize(candidate, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
-
-                        } else {
-                            routingNodes.initialize(candidate, minNode.getNodeId(), null, allocation.clusterInfo().getShardSize(candidate, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
-                        }
+                        /* now allocate on the cluster */
+                        routingNodes.relocate(candidate, minNode.getNodeId(), allocation.clusterInfo().getShardSize(candidate, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
                         return true;
-
                     }
                 }
             }
@@ -846,14 +802,12 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
     }
 
     static class ModelNode implements Iterable<ModelIndex> {
-        private final String id;
         private final Map<String, ModelIndex> indices = new HashMap<>();
         private int numShards = 0;
-        // lazily calculated
-        private RoutingNode routingNode;
+        private final RoutingNode routingNode;
 
-        public ModelNode(String id) {
-            this.id = id;
+        public ModelNode(RoutingNode routingNode) {
+            this.routingNode = routingNode;
         }
 
         public ModelIndex getIndex(String indexId) {
@@ -861,13 +815,10 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
         }
 
         public String getNodeId() {
-            return id;
+            return routingNode.nodeId();
         }
 
-        public RoutingNode getRoutingNode(RoutingNodes routingNodes) {
-            if (routingNode == null) {
-                routingNode = routingNodes.node(id);
-            }
+        public RoutingNode getRoutingNode() {
             return routingNode;
         }
 
@@ -888,33 +839,31 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
             return -1;
         }
 
-        public void addShard(ShardRouting shard, Decision decision) {
+        public void addShard(ShardRouting shard) {
             ModelIndex index = indices.get(shard.getIndexName());
             if (index == null) {
                 index = new ModelIndex(shard.getIndexName());
                 indices.put(index.getIndexId(), index);
             }
-            index.addShard(shard, decision);
+            index.addShard(shard);
             numShards++;
         }
 
-        public Decision removeShard(ShardRouting shard) {
+        public void removeShard(ShardRouting shard) {
             ModelIndex index = indices.get(shard.getIndexName());
-            Decision removed = null;
             if (index != null) {
-                removed = index.removeShard(shard);
-                if (removed != null && index.numShards() == 0) {
+                index.removeShard(shard);
+                if (index.numShards() == 0) {
                     indices.remove(shard.getIndexName());
                 }
             }
             numShards--;
-            return removed;
         }
 
         @Override
         public String toString() {
             StringBuilder sb = new StringBuilder();
-            sb.append("Node(").append(id).append(")");
+            sb.append("Node(").append(routingNode.nodeId()).append(")");
             return sb.toString();
         }
 
@@ -930,9 +879,9 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
 
     }
 
-    static final class ModelIndex {
+    static final class ModelIndex implements Iterable<ShardRouting> {
         private final String id;
-        private final Map<ShardRouting, Decision> shards = new HashMap<>();
+        private final Set<ShardRouting> shards = new HashSet<>(4); // expect few shards of same index to be allocated on same node
         private int highestPrimary = -1;
 
         public ModelIndex(String id) {
@@ -942,7 +891,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
         public int highestPrimary() {
             if (highestPrimary == -1) {
                 int maxId = -1;
-                for (ShardRouting shard : shards.keySet()) {
+                for (ShardRouting shard : shards) {
                     if (shard.primary()) {
                         maxId = Math.max(maxId, shard.id());
                     }
@@ -960,24 +909,25 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
             return shards.size();
         }
 
-        public Collection<ShardRouting> getAllShards() {
-            return shards.keySet();
+        @Override
+        public Iterator<ShardRouting> iterator() {
+            return shards.iterator();
         }
 
-        public Decision removeShard(ShardRouting shard) {
+        public void removeShard(ShardRouting shard) {
             highestPrimary = -1;
-            return shards.remove(shard);
+            assert shards.contains(shard) : "Shard not allocated on current node: " + shard;
+            shards.remove(shard);
         }
 
-        public void addShard(ShardRouting shard, Decision decision) {
+        public void addShard(ShardRouting shard) {
             highestPrimary = -1;
-            assert decision != null;
-            assert !shards.containsKey(shard) : "Shard already allocated on current node: " + shards.get(shard) + " " + shard;
-            shards.put(shard, decision);
+            assert !shards.contains(shard) : "Shard already allocated on current node: " + shard;
+            shards.add(shard);
         }
 
         public boolean containsShard(ShardRouting shard) {
-            return shards.containsKey(shard);
+            return shards.contains(shard);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocator.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocator.java
index 4d9c05527d3..0bf07e8cba9 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocator.java
@@ -19,56 +19,25 @@
 
 package org.elasticsearch.cluster.routing.allocation.allocator;
 
-import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.ShardRoutingState;
-import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation;
 
 /**
  * <p>
  * A {@link ShardsAllocator} is the main entry point for shard allocation on nodes in the cluster.
  * The allocator makes basic decision where a shard instance will be allocated, if already allocated instances
- * need relocate to other nodes due to node failures or due to rebalancing decisions.
+ * need to relocate to other nodes due to node failures or due to rebalancing decisions.
  * </p>
  */
 public interface ShardsAllocator {
 
     /**
-     * Applies changes on started nodes based on the implemented algorithm. For example if a
-     * shard has changed to {@link ShardRoutingState#STARTED} from {@link ShardRoutingState#RELOCATING}
-     * this allocator might apply some cleanups on the node that used to hold the shard.
-     * @param allocation all started {@link ShardRouting shards}
-     */
-    void applyStartedShards(StartedRerouteAllocation allocation);
-
-    /**
-     * Applies changes on failed nodes based on the implemented algorithm.
-     * @param allocation all failed {@link ShardRouting shards}
-     */
-    void applyFailedShards(FailedRerouteAllocation allocation);
-
-    /**
-     * Assign all unassigned shards to nodes
+     * Allocates shards to nodes in the cluster. An implementation of this method should:
+     * - assign unassigned shards
+     * - relocate shards that cannot stay on a node anymore
+     * - relocate shards to find a good shard balance in the cluster
      *
      * @param allocation current node allocation
      * @return <code>true</code> if the allocation has changed, otherwise <code>false</code>
      */
-    boolean allocateUnassigned(RoutingAllocation allocation);
-
-    /**
-     * Rebalancing number of shards on all nodes
-     *
-     * @param allocation current node allocation
-     * @return <code>true</code> if the allocation has changed, otherwise <code>false</code>
-     */
-    boolean rebalance(RoutingAllocation allocation);
-
-    /**
-     * Move started shards that can not be allocated to a node anymore
-     *
-     * @param allocation current node allocation
-     * @return <code>true</code> if the allocation has changed, otherwise <code>false</code>
-     */
-    boolean moveShards(RoutingAllocation allocation);
+    boolean allocate(RoutingAllocation allocation);
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocators.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocators.java
deleted file mode 100644
index f3eb1ebbf14..00000000000
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocators.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster.routing.allocation.allocator;
-
-import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.gateway.GatewayAllocator;
-
-/**
- * The {@link ShardsAllocator} class offers methods for allocating shard within a cluster.
- * These methods include moving shards and re-balancing the cluster. It also allows management
- * of shards by their state.
- */
-public class ShardsAllocators extends AbstractComponent implements ShardsAllocator {
-
-    private final GatewayAllocator gatewayAllocator;
-    private final ShardsAllocator allocator;
-
-    public ShardsAllocators(GatewayAllocator allocator) {
-        this(Settings.Builder.EMPTY_SETTINGS, allocator);
-    }
-
-    public ShardsAllocators(Settings settings, GatewayAllocator allocator) {
-      this(settings, allocator, new BalancedShardsAllocator(settings));
-    }
-
-    @Inject
-    public ShardsAllocators(Settings settings, GatewayAllocator gatewayAllocator, ShardsAllocator allocator) {
-        super(settings);
-        this.gatewayAllocator = gatewayAllocator;
-        this.allocator = allocator;
-    }
-
-    @Override
-    public void applyStartedShards(StartedRerouteAllocation allocation) {
-        gatewayAllocator.applyStartedShards(allocation);
-        allocator.applyStartedShards(allocation);
-    }
-
-    @Override
-    public void applyFailedShards(FailedRerouteAllocation allocation) {
-        gatewayAllocator.applyFailedShards(allocation);
-        allocator.applyFailedShards(allocation);
-    }
-
-    @Override
-    public boolean allocateUnassigned(RoutingAllocation allocation) {
-        boolean changed = false;
-        changed |= gatewayAllocator.allocateUnassigned(allocation);
-        changed |= allocator.allocateUnassigned(allocation);
-        return changed;
-    }
-
-    protected long nanoTime() {
-        return System.nanoTime();
-    }
-
-    @Override
-    public boolean rebalance(RoutingAllocation allocation) {
-        if (allocation.hasPendingAsyncFetch() == false) {
-            /*
-             * see https://github.com/elastic/elasticsearch/issues/14387
-             * if we allow rebalance operations while we are still fetching shard store data
-             * we might end up with unnecessary rebalance operations which can be super confusion/frustrating
-             * since once the fetches come back we might just move all the shards back again.
-             * Therefore we only do a rebalance if we have fetched all information.
-             */
-            return allocator.rebalance(allocation);
-        } else {
-            logger.debug("skipping rebalance due to in-flight shard/store fetches");
-            return false;
-        }
-    }
-
-    @Override
-    public boolean moveShards(RoutingAllocation allocation) {
-        return allocator.moveShards(allocation);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java
index 5ccd9e9bb63..f4b1be19af4 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java
@@ -112,7 +112,7 @@ public class AllocateStalePrimaryAllocationCommand extends BasePrimaryAllocation
                 "allocating an empty primary for [" + index + "][" + shardId + "] can result in data loss. Please confirm by setting the accept_data_loss parameter to true");
         }
 
-        final IndexMetaData indexMetaData = allocation.metaData().index(shardRouting.getIndexName());
+        final IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardRouting.index());
         if (shardRouting.allocatedPostIndexCreate(indexMetaData) == false) {
             return explainOrThrowRejectedCommand(explain, allocation,
                 "trying to allocate an existing primary shard [" + index + "][" + shardId + "], while no such shard has ever been active");
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
index 9859a9b6584..227ec277469 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
@@ -28,6 +28,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 import java.util.HashMap;
@@ -77,8 +78,11 @@ public class AwarenessAllocationDecider extends AllocationDecider {
 
     public static final String NAME = "awareness";
 
-    public static final Setting<String[]> CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING = new Setting<>("cluster.routing.allocation.awareness.attributes", "", Strings::splitStringByCommaToArray , true, Setting.Scope.CLUSTER);
-    public static final Setting<Settings> CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING = Setting.groupSetting("cluster.routing.allocation.awareness.force.", true, Setting.Scope.CLUSTER);
+    public static final Setting<String[]> CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING =
+        new Setting<>("cluster.routing.allocation.awareness.attributes", "", Strings::splitStringByCommaToArray , Property.Dynamic,
+            Property.NodeScope);
+    public static final Setting<Settings> CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING =
+        Setting.groupSetting("cluster.routing.allocation.awareness.force.", Property.Dynamic, Property.NodeScope);
 
     private String[] awarenessAttributes;
 
@@ -149,7 +153,7 @@ public class AwarenessAllocationDecider extends AllocationDecider {
             return allocation.decision(Decision.YES, NAME, "no allocation awareness enabled");
         }
 
-        IndexMetaData indexMetaData = allocation.metaData().index(shardRouting.index());
+        IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardRouting.index());
         int shardCount = indexMetaData.getNumberOfReplicas() + 1; // 1 for primary
         for (String awarenessAttribute : awarenessAttributes) {
             // the node the shard exists on must be associated with an awareness attribute
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java
index 11fce397b26..84e974aceb0 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java
@@ -24,6 +24,7 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 import java.util.Locale;
@@ -48,7 +49,9 @@ import java.util.Locale;
 public class ClusterRebalanceAllocationDecider extends AllocationDecider {
 
     public static final String NAME = "cluster_rebalance";
-    public static final Setting<ClusterRebalanceType> CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING = new Setting<>("cluster.routing.allocation.allow_rebalance", ClusterRebalanceType.INDICES_ALL_ACTIVE.name().toLowerCase(Locale.ROOT), ClusterRebalanceType::parseString, true, Setting.Scope.CLUSTER);
+    public static final Setting<ClusterRebalanceType> CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING =
+        new Setting<>("cluster.routing.allocation.allow_rebalance", ClusterRebalanceType.INDICES_ALL_ACTIVE.name().toLowerCase(Locale.ROOT),
+            ClusterRebalanceType::parseString, Property.Dynamic, Property.NodeScope);
 
     /**
      * An enum representation for the configured re-balance type.
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java
index d39b9604066..fe6bf918dc2 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java
@@ -24,6 +24,7 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 /**
@@ -42,7 +43,9 @@ public class ConcurrentRebalanceAllocationDecider extends AllocationDecider {
 
     public static final String NAME = "concurrent_rebalance";
 
-    public static final Setting<Integer> CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING = Setting.intSetting("cluster.routing.allocation.cluster_concurrent_rebalance", 2, -1, true, Setting.Scope.CLUSTER);
+    public static final Setting<Integer> CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING =
+        Setting.intSetting("cluster.routing.allocation.cluster_concurrent_rebalance", 2, -1,
+            Property.Dynamic, Property.NodeScope);
     private volatile int clusterConcurrentRebalance;
 
     @Inject
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
index 821fa55d704..e2124558f2d 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
@@ -38,6 +38,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.RatioValue;
@@ -81,11 +82,22 @@ public class DiskThresholdDecider extends AllocationDecider {
     private volatile boolean enabled;
     private volatile TimeValue rerouteInterval;
 
-    public static final Setting<Boolean> CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING = Setting.boolSetting("cluster.routing.allocation.disk.threshold_enabled", true, true, Setting.Scope.CLUSTER);
-    public static final Setting<String> CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING = new Setting<>("cluster.routing.allocation.disk.watermark.low", "85%", (s) -> validWatermarkSetting(s, "cluster.routing.allocation.disk.watermark.low"), true, Setting.Scope.CLUSTER);
-    public static final Setting<String> CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING = new Setting<>("cluster.routing.allocation.disk.watermark.high", "90%", (s) -> validWatermarkSetting(s, "cluster.routing.allocation.disk.watermark.high"), true, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS_SETTING = Setting.boolSetting("cluster.routing.allocation.disk.include_relocations", true, true, Setting.Scope.CLUSTER);;
-    public static final Setting<TimeValue> CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING = Setting.positiveTimeSetting("cluster.routing.allocation.disk.reroute_interval", TimeValue.timeValueSeconds(60), true, Setting.Scope.CLUSTER);
+    public static final Setting<Boolean> CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING =
+        Setting.boolSetting("cluster.routing.allocation.disk.threshold_enabled", true, Property.Dynamic, Property.NodeScope);
+    public static final Setting<String> CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING =
+        new Setting<>("cluster.routing.allocation.disk.watermark.low", "85%",
+            (s) -> validWatermarkSetting(s, "cluster.routing.allocation.disk.watermark.low"),
+            Property.Dynamic, Property.NodeScope);
+    public static final Setting<String> CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING =
+        new Setting<>("cluster.routing.allocation.disk.watermark.high", "90%",
+            (s) -> validWatermarkSetting(s, "cluster.routing.allocation.disk.watermark.high"),
+            Property.Dynamic, Property.NodeScope);
+    public static final Setting<Boolean> CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS_SETTING =
+        Setting.boolSetting("cluster.routing.allocation.disk.include_relocations", true,
+            Property.Dynamic, Property.NodeScope);;
+    public static final Setting<TimeValue> CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING =
+        Setting.positiveTimeSetting("cluster.routing.allocation.disk.reroute_interval", TimeValue.timeValueSeconds(60),
+            Property.Dynamic, Property.NodeScope);
 
     /**
      * Listens for a node to go over the high watermark and kicks off an empty
@@ -330,7 +342,7 @@ public class DiskThresholdDecider extends AllocationDecider {
         }
 
         // a flag for whether the primary shard has been previously allocated
-        IndexMetaData indexMetaData = allocation.metaData().index(shardRouting.getIndexName());
+        IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardRouting.index());
         boolean primaryHasBeenAllocated = shardRouting.primary() && shardRouting.allocatedPostIndexCreate(indexMetaData);
 
         // checks for exact byte comparisons
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
index 9131355876b..0b69ba2a19e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
@@ -26,6 +26,7 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 import java.util.Locale;
@@ -60,11 +61,19 @@ public class EnableAllocationDecider extends AllocationDecider {
 
     public static final String NAME = "enable";
 
-    public static final Setting<Allocation> CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING = new Setting<>("cluster.routing.allocation.enable", Allocation.ALL.name(), Allocation::parse, true, Setting.Scope.CLUSTER);
-    public static final Setting<Allocation> INDEX_ROUTING_ALLOCATION_ENABLE_SETTING = new Setting<>("index.routing.allocation.enable", Allocation.ALL.name(), Allocation::parse, true, Setting.Scope.INDEX);
+    public static final Setting<Allocation> CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING =
+        new Setting<>("cluster.routing.allocation.enable", Allocation.ALL.name(), Allocation::parse,
+            Property.Dynamic, Property.NodeScope);
+    public static final Setting<Allocation> INDEX_ROUTING_ALLOCATION_ENABLE_SETTING =
+        new Setting<>("index.routing.allocation.enable", Allocation.ALL.name(), Allocation::parse,
+            Property.Dynamic, Property.IndexScope);
 
-    public static final Setting<Rebalance> CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING = new Setting<>("cluster.routing.rebalance.enable", Rebalance.ALL.name(), Rebalance::parse, true, Setting.Scope.CLUSTER);
-    public static final Setting<Rebalance> INDEX_ROUTING_REBALANCE_ENABLE_SETTING = new Setting<>("index.routing.rebalance.enable", Rebalance.ALL.name(), Rebalance::parse, true, Setting.Scope.INDEX);
+    public static final Setting<Rebalance> CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING =
+        new Setting<>("cluster.routing.rebalance.enable", Rebalance.ALL.name(), Rebalance::parse,
+            Property.Dynamic, Property.NodeScope);
+    public static final Setting<Rebalance> INDEX_ROUTING_REBALANCE_ENABLE_SETTING =
+        new Setting<>("index.routing.rebalance.enable", Rebalance.ALL.name(), Rebalance::parse,
+            Property.Dynamic, Property.IndexScope);
 
     private volatile Rebalance enableRebalance;
     private volatile Allocation enableAllocation;
@@ -92,7 +101,7 @@ public class EnableAllocationDecider extends AllocationDecider {
             return allocation.decision(Decision.YES, NAME, "allocation disabling is ignored");
         }
 
-        final IndexMetaData indexMetaData = allocation.metaData().index(shardRouting.getIndexName());
+        final IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardRouting.index());
         final Allocation enable;
         if (INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.exists(indexMetaData.getSettings())) {
             enable = INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.get(indexMetaData.getSettings());
@@ -127,7 +136,7 @@ public class EnableAllocationDecider extends AllocationDecider {
             return allocation.decision(Decision.YES, NAME, "rebalance disabling is ignored");
         }
 
-        Settings indexSettings = allocation.routingNodes().metaData().index(shardRouting.index()).getSettings();
+        Settings indexSettings = allocation.routingNodes().metaData().getIndexSafe(shardRouting.index()).getSettings();
         final Rebalance enable;
         if (INDEX_ROUTING_REBALANCE_ENABLE_SETTING.exists(indexSettings)) {
             enable = INDEX_ROUTING_REBALANCE_ENABLE_SETTING.get(indexSettings);
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
index f8ff5f37aed..d1aa0d8b583 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
@@ -27,6 +27,7 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.AND;
@@ -60,9 +61,12 @@ public class FilterAllocationDecider extends AllocationDecider {
 
     public static final String NAME = "filter";
 
-    public static final Setting<Settings> CLUSTER_ROUTING_REQUIRE_GROUP_SETTING = Setting.groupSetting("cluster.routing.allocation.require.", true, Setting.Scope.CLUSTER);
-    public static final Setting<Settings> CLUSTER_ROUTING_INCLUDE_GROUP_SETTING = Setting.groupSetting("cluster.routing.allocation.include.", true, Setting.Scope.CLUSTER);
-    public static final Setting<Settings> CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING = Setting.groupSetting("cluster.routing.allocation.exclude.", true, Setting.Scope.CLUSTER);
+    public static final Setting<Settings> CLUSTER_ROUTING_REQUIRE_GROUP_SETTING =
+        Setting.groupSetting("cluster.routing.allocation.require.", Property.Dynamic, Property.NodeScope);
+    public static final Setting<Settings> CLUSTER_ROUTING_INCLUDE_GROUP_SETTING =
+        Setting.groupSetting("cluster.routing.allocation.include.", Property.Dynamic, Property.NodeScope);
+    public static final Setting<Settings> CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING =
+        Setting.groupSetting("cluster.routing.allocation.exclude.", Property.Dynamic, Property.NodeScope);
 
     private volatile DiscoveryNodeFilters clusterRequireFilters;
     private volatile DiscoveryNodeFilters clusterIncludeFilters;
@@ -98,7 +102,7 @@ public class FilterAllocationDecider extends AllocationDecider {
         Decision decision = shouldClusterFilter(node, allocation);
         if (decision != null) return decision;
 
-        decision = shouldIndexFilter(allocation.routingNodes().metaData().index(shardRouting.index()), node, allocation);
+        decision = shouldIndexFilter(allocation.routingNodes().metaData().getIndexSafe(shardRouting.index()), node, allocation);
         if (decision != null) return decision;
 
         return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require filters");
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
index e766b4c49aa..04247525f1d 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
@@ -27,6 +27,7 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 /**
@@ -59,13 +60,17 @@ public class ShardsLimitAllocationDecider extends AllocationDecider {
      * Controls the maximum number of shards per index on a single Elasticsearch
      * node. Negative values are interpreted as unlimited.
      */
-    public static final Setting<Integer> INDEX_TOTAL_SHARDS_PER_NODE_SETTING = Setting.intSetting("index.routing.allocation.total_shards_per_node", -1, -1, true, Setting.Scope.INDEX);
+    public static final Setting<Integer> INDEX_TOTAL_SHARDS_PER_NODE_SETTING =
+        Setting.intSetting("index.routing.allocation.total_shards_per_node", -1, -1,
+            Property.Dynamic, Property.IndexScope);
 
     /**
      * Controls the maximum number of shards per node on a global level.
      * Negative values are interpreted as unlimited.
      */
-    public static final Setting<Integer> CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING = Setting.intSetting("cluster.routing.allocation.total_shards_per_node", -1,  -1, true, Setting.Scope.CLUSTER);
+    public static final Setting<Integer> CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING =
+        Setting.intSetting("cluster.routing.allocation.total_shards_per_node", -1,  -1,
+            Property.Dynamic, Property.NodeScope);
 
 
     @Inject
@@ -81,7 +86,7 @@ public class ShardsLimitAllocationDecider extends AllocationDecider {
 
     @Override
     public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
-        IndexMetaData indexMd = allocation.routingNodes().metaData().index(shardRouting.index());
+        IndexMetaData indexMd = allocation.routingNodes().metaData().getIndexSafe(shardRouting.index());
         final int indexShardLimit = INDEX_TOTAL_SHARDS_PER_NODE_SETTING.get(indexMd.getSettings(), settings);
         // Capture the limit here in case it changes during this method's
         // execution
@@ -118,7 +123,7 @@ public class ShardsLimitAllocationDecider extends AllocationDecider {
 
     @Override
     public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
-        IndexMetaData indexMd = allocation.routingNodes().metaData().index(shardRouting.index());
+        IndexMetaData indexMd = allocation.routingNodes().metaData().getIndexSafe(shardRouting.index());
         final int indexShardLimit = INDEX_TOTAL_SHARDS_PER_NODE_SETTING.get(indexMd.getSettings(), settings);
         // Capture the limit here in case it changes during this method's
         // execution
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java
index cf889cde6ad..d656afc8036 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java
@@ -26,6 +26,7 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 /**
@@ -39,7 +40,9 @@ public class SnapshotInProgressAllocationDecider extends AllocationDecider {
     /**
      * Disables relocation of shards that are currently being snapshotted.
      */
-    public static final Setting<Boolean> CLUSTER_ROUTING_ALLOCATION_SNAPSHOT_RELOCATION_ENABLED_SETTING = Setting.boolSetting("cluster.routing.allocation.snapshot.relocation_enabled", false, true, Setting.Scope.CLUSTER);
+    public static final Setting<Boolean> CLUSTER_ROUTING_ALLOCATION_SNAPSHOT_RELOCATION_ENABLED_SETTING =
+        Setting.boolSetting("cluster.routing.allocation.snapshot.relocation_enabled", false,
+            Property.Dynamic, Property.NodeScope);
 
     private volatile boolean enableRelocation = false;
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
index 25f43f57610..ca6b312da4c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
@@ -25,6 +25,7 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 /**
@@ -50,10 +51,25 @@ public class ThrottlingAllocationDecider extends AllocationDecider {
     public static final int DEFAULT_CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES = 2;
     public static final int DEFAULT_CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES = 4;
     public static final String NAME = "throttling";
-    public static final Setting<Integer> CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING = new Setting<>("cluster.routing.allocation.node_concurrent_recoveries", Integer.toString(DEFAULT_CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES), (s) -> Setting.parseInt(s, 0, "cluster.routing.allocation.node_concurrent_recoveries"), true, Setting.Scope.CLUSTER);
-    public static final Setting<Integer> CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING = Setting.intSetting("cluster.routing.allocation.node_initial_primaries_recoveries", DEFAULT_CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES, 0, true, Setting.Scope.CLUSTER);
-    public static final Setting<Integer> CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING = new Setting<>("cluster.routing.allocation.node_concurrent_incoming_recoveries", (s) -> CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING.getRaw(s), (s) -> Setting.parseInt(s, 0, "cluster.routing.allocation.node_concurrent_incoming_recoveries"), true, Setting.Scope.CLUSTER);
-    public static final Setting<Integer> CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING = new Setting<>("cluster.routing.allocation.node_concurrent_outgoing_recoveries", (s) -> CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING.getRaw(s), (s) -> Setting.parseInt(s, 0, "cluster.routing.allocation.node_concurrent_outgoing_recoveries"), true, Setting.Scope.CLUSTER);
+    public static final Setting<Integer> CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING =
+        new Setting<>("cluster.routing.allocation.node_concurrent_recoveries",
+            Integer.toString(DEFAULT_CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES),
+            (s) -> Setting.parseInt(s, 0, "cluster.routing.allocation.node_concurrent_recoveries"),
+            Property.Dynamic, Property.NodeScope);
+    public static final Setting<Integer> CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING =
+        Setting.intSetting("cluster.routing.allocation.node_initial_primaries_recoveries",
+            DEFAULT_CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES, 0,
+            Property.Dynamic, Property.NodeScope);
+    public static final Setting<Integer> CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING =
+        new Setting<>("cluster.routing.allocation.node_concurrent_incoming_recoveries",
+            (s) -> CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING.getRaw(s),
+            (s) -> Setting.parseInt(s, 0, "cluster.routing.allocation.node_concurrent_incoming_recoveries"),
+            Property.Dynamic, Property.NodeScope);
+    public static final Setting<Integer> CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING =
+        new Setting<>("cluster.routing.allocation.node_concurrent_outgoing_recoveries",
+            (s) -> CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING.getRaw(s),
+            (s) -> Setting.parseInt(s, 0, "cluster.routing.allocation.node_concurrent_outgoing_recoveries"),
+            Property.Dynamic, Property.NodeScope);
 
 
     private volatile int primariesInitialRecoveries;
diff --git a/core/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
similarity index 75%
rename from core/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java
rename to core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
index df22bdb93e5..d3fca22c165 100644
--- a/core/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
@@ -19,11 +19,9 @@
 
 package org.elasticsearch.cluster.service;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AckedClusterStateTaskListener;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterState.Builder;
 import org.elasticsearch.cluster.ClusterStateListener;
@@ -32,19 +30,18 @@ import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.LocalNodeMasterListener;
+import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.TimeoutClusterStateListener;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.ProcessClusterEventTimeoutException;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.OperationRouting;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
@@ -52,9 +49,9 @@ import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.CountDown;
@@ -65,9 +62,7 @@ import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
 import org.elasticsearch.common.util.concurrent.PrioritizedRunnable;
 import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.discovery.Discovery;
-import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -78,8 +73,6 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Queue;
-import java.util.Random;
-import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.Executor;
 import java.util.concurrent.Future;
@@ -94,28 +87,20 @@ import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadF
 /**
  *
  */
-public class InternalClusterService extends AbstractLifecycleComponent<ClusterService> implements ClusterService {
+public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
 
-    public static final Setting<TimeValue> CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING = Setting.positiveTimeSetting("cluster.service.slow_task_logging_threshold", TimeValue.timeValueSeconds(30), true, Setting.Scope.CLUSTER);
-    public static final Setting<TimeValue> CLUSTER_SERVICE_RECONNECT_INTERVAL_SETTING = Setting.positiveTimeSetting("cluster.service.reconnect_interval",  TimeValue.timeValueSeconds(10), false, Setting.Scope.CLUSTER);
+    public static final Setting<TimeValue> CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING =
+            Setting.positiveTimeSetting("cluster.service.slow_task_logging_threshold", TimeValue.timeValueSeconds(30),
+                    Property.Dynamic, Property.NodeScope);
 
     public static final String UPDATE_THREAD_NAME = "clusterService#updateTask";
-    public static final Setting<Long> NODE_ID_SEED_SETTING =
-            // don't use node.id.seed so it won't be seen as an attribute
-            Setting.longSetting("node_id.seed", 0L, Long.MIN_VALUE, false, Setting.Scope.CLUSTER);
     private final ThreadPool threadPool;
 
     private BiConsumer<ClusterChangedEvent, Discovery.AckListener> clusterStatePublisher;
 
     private final OperationRouting operationRouting;
 
-    private final TransportService transportService;
-
     private final ClusterSettings clusterSettings;
-    private final DiscoveryNodeService discoveryNodeService;
-    private final Version version;
-
-    private final TimeValue reconnectInterval;
 
     private TimeValue slowTaskLoggingThreshold;
 
@@ -130,7 +115,8 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
     private final Map<ClusterStateTaskExecutor, List<UpdateTask>> updateTasksPerExecutor = new HashMap<>();
     // TODO this is rather frequently changing I guess a Synced Set would be better here and a dedicated remove API
     private final Collection<ClusterStateListener> postAppliedListeners = new CopyOnWriteArrayList<>();
-    private final Iterable<ClusterStateListener> preAppliedListeners = Iterables.concat(priorityClusterStateListeners, clusterStateListeners, lastClusterStateListeners);
+    private final Iterable<ClusterStateListener> preAppliedListeners = Iterables.concat(priorityClusterStateListeners,
+            clusterStateListeners, lastClusterStateListeners);
 
     private final LocalNodeMasterListeners localNodeMasterListeners;
 
@@ -140,60 +126,69 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
 
     private final ClusterBlocks.Builder initialBlocks;
 
-    private final TaskManager taskManager;
-
-    private volatile ScheduledFuture reconnectToNodes;
+    private NodeConnectionsService nodeConnectionsService;
 
     @Inject
-    public InternalClusterService(Settings settings, OperationRouting operationRouting, TransportService transportService,
-                                  ClusterSettings clusterSettings, ThreadPool threadPool, ClusterName clusterName, DiscoveryNodeService discoveryNodeService, Version version) {
+    public ClusterService(Settings settings, OperationRouting operationRouting,
+                          ClusterSettings clusterSettings, ThreadPool threadPool, ClusterName clusterName) {
         super(settings);
         this.operationRouting = operationRouting;
-        this.transportService = transportService;
         this.threadPool = threadPool;
         this.clusterSettings = clusterSettings;
-        this.discoveryNodeService = discoveryNodeService;
-        this.version = version;
 
         // will be replaced on doStart.
         this.clusterState = ClusterState.builder(clusterName).build();
 
-        this.clusterSettings.addSettingsUpdateConsumer(CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING, this::setSlowTaskLoggingThreshold);
-
-        this.reconnectInterval = CLUSTER_SERVICE_RECONNECT_INTERVAL_SETTING.get(settings);
+        this.clusterSettings.addSettingsUpdateConsumer(CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING,
+                this::setSlowTaskLoggingThreshold);
 
         this.slowTaskLoggingThreshold = CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get(settings);
 
         localNodeMasterListeners = new LocalNodeMasterListeners(threadPool);
 
         initialBlocks = ClusterBlocks.builder();
-
-        taskManager = transportService.getTaskManager();
     }
 
     private void setSlowTaskLoggingThreshold(TimeValue slowTaskLoggingThreshold) {
         this.slowTaskLoggingThreshold = slowTaskLoggingThreshold;
     }
 
-    public void setClusterStatePublisher(BiConsumer<ClusterChangedEvent, Discovery.AckListener> publisher) {
+    synchronized public void setClusterStatePublisher(BiConsumer<ClusterChangedEvent, Discovery.AckListener> publisher) {
         clusterStatePublisher = publisher;
     }
 
-    @Override
-    public void addInitialStateBlock(ClusterBlock block) throws IllegalStateException {
+    synchronized public void setLocalNode(DiscoveryNode localNode) {
+        assert clusterState.nodes().localNodeId() == null : "local node is already set";
+        DiscoveryNodes.Builder nodeBuilder = DiscoveryNodes.builder(clusterState.nodes()).put(localNode).localNodeId(localNode.id());
+        this.clusterState = ClusterState.builder(clusterState).nodes(nodeBuilder).build();
+    }
+
+    synchronized public void setNodeConnectionsService(NodeConnectionsService nodeConnectionsService) {
+        assert this.nodeConnectionsService == null : "nodeConnectionsService is already set";
+        this.nodeConnectionsService = nodeConnectionsService;
+    }
+
+    /**
+     * Adds an initial block to be set on the first cluster state created.
+     */
+    synchronized public void addInitialStateBlock(ClusterBlock block) throws IllegalStateException {
         if (lifecycle.started()) {
             throw new IllegalStateException("can't set initial block when started");
         }
         initialBlocks.addGlobalBlock(block);
     }
 
-    @Override
-    public void removeInitialStateBlock(ClusterBlock block) throws IllegalStateException {
+    /**
+     * Remove an initial block to be set on the first cluster state created.
+     */
+    synchronized public void removeInitialStateBlock(ClusterBlock block) throws IllegalStateException {
         removeInitialStateBlock(block.id());
     }
 
-    @Override
-    public void removeInitialStateBlock(int blockId) throws IllegalStateException {
+    /**
+     * Remove an initial block to be set on the first cluster state created.
+     */
+    synchronized public void removeInitialStateBlock(int blockId) throws IllegalStateException {
         if (lifecycle.started()) {
             throw new IllegalStateException("can't set initial block when started");
         }
@@ -201,28 +196,27 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
     }
 
     @Override
-    protected void doStart() {
+    synchronized protected void doStart() {
         Objects.requireNonNull(clusterStatePublisher, "please set a cluster state publisher before starting");
+        Objects.requireNonNull(clusterState.nodes().localNode(), "please set the local node before starting");
+        Objects.requireNonNull(nodeConnectionsService, "please set the node connection service before starting");
         add(localNodeMasterListeners);
-        add(taskManager);
         this.clusterState = ClusterState.builder(clusterState).blocks(initialBlocks).build();
-        this.updateTasksExecutor = EsExecutors.newSinglePrioritizing(UPDATE_THREAD_NAME, daemonThreadFactory(settings, UPDATE_THREAD_NAME), threadPool.getThreadContext());
-        this.reconnectToNodes = threadPool.schedule(reconnectInterval, ThreadPool.Names.GENERIC, new ReconnectToNodes());
-        Map<String, String> nodeAttributes = discoveryNodeService.buildAttributes();
-        // note, we rely on the fact that its a new id each time we start, see FD and "kill -9" handling
-        final String nodeId = generateNodeId(settings);
-        final TransportAddress publishAddress = transportService.boundAddress().publishAddress();
-        DiscoveryNode localNode = new DiscoveryNode(settings.get("node.name"), nodeId, publishAddress, nodeAttributes, version);
-        DiscoveryNodes.Builder nodeBuilder = DiscoveryNodes.builder().put(localNode).localNodeId(localNode.id());
-        this.clusterState = ClusterState.builder(clusterState).nodes(nodeBuilder).blocks(initialBlocks).build();
-        this.transportService.setLocalNode(localNode);
+        this.updateTasksExecutor = EsExecutors.newSinglePrioritizing(UPDATE_THREAD_NAME, daemonThreadFactory(settings, UPDATE_THREAD_NAME),
+                threadPool.getThreadContext());
+        this.clusterState = ClusterState.builder(clusterState).blocks(initialBlocks).build();
     }
 
     @Override
-    protected void doStop() {
-        FutureUtils.cancel(this.reconnectToNodes);
+    synchronized protected void doStop() {
         for (NotifyTimeout onGoingTimeout : onGoingTimeouts) {
             onGoingTimeout.cancel();
+            try {
+                onGoingTimeout.cancel();
+                onGoingTimeout.listener.onClose();
+            } catch (Exception ex) {
+                logger.debug("failed to notify listeners on shutdown", ex);
+            }
         }
         ThreadPool.terminate(updateTasksExecutor, 10, TimeUnit.SECONDS);
         postAppliedListeners.stream().filter(listener -> listener instanceof TimeoutClusterStateListener)
@@ -231,40 +225,51 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
     }
 
     @Override
-    protected void doClose() {
+    synchronized protected void doClose() {
     }
 
-    @Override
+    /**
+     * The local node.
+     */
     public DiscoveryNode localNode() {
         return clusterState.getNodes().localNode();
     }
 
-    @Override
     public OperationRouting operationRouting() {
         return operationRouting;
     }
 
-    @Override
+    /**
+     * The current state.
+     */
     public ClusterState state() {
         return this.clusterState;
     }
 
-    @Override
+    /**
+     * Adds a priority listener for updated cluster states.
+     */
     public void addFirst(ClusterStateListener listener) {
         priorityClusterStateListeners.add(listener);
     }
 
-    @Override
+    /**
+     * Adds last listener.
+     */
     public void addLast(ClusterStateListener listener) {
         lastClusterStateListeners.add(listener);
     }
 
-    @Override
+    /**
+     * Adds a listener for updated cluster states.
+     */
     public void add(ClusterStateListener listener) {
         clusterStateListeners.add(listener);
     }
 
-    @Override
+    /**
+     * Removes a listener for updated cluster states.
+     */
     public void remove(ClusterStateListener listener) {
         clusterStateListeners.remove(listener);
         priorityClusterStateListeners.remove(listener);
@@ -279,17 +284,27 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
         }
     }
 
-    @Override
+    /**
+     * Add a listener for on/off local node master events
+     */
     public void add(LocalNodeMasterListener listener) {
         localNodeMasterListeners.add(listener);
     }
 
-    @Override
+    /**
+     * Remove the given listener for on/off local master events
+     */
     public void remove(LocalNodeMasterListener listener) {
         localNodeMasterListeners.remove(listener);
     }
 
-    @Override
+    /**
+     * Adds a cluster state listener that will timeout after the provided timeout,
+     * and is executed after the clusterstate has been successfully applied ie. is
+     * in state {@link org.elasticsearch.cluster.ClusterState.ClusterStateStatus#APPLIED}
+     * NOTE: a {@code null} timeout means that the listener will never be removed
+     * automatically
+     */
     public void add(@Nullable final TimeValue timeout, final TimeoutClusterStateListener listener) {
         if (lifecycle.stoppedOrClosed()) {
             listener.onClose();
@@ -318,13 +333,37 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
         }
     }
 
-    @Override
+    /**
+     * Submits a cluster state update task; unlike {@link #submitStateUpdateTask(String, Object, ClusterStateTaskConfig,
+     * ClusterStateTaskExecutor, ClusterStateTaskListener)}, submitted updates will not be batched.
+     *
+     * @param source     the source of the cluster state update task
+     * @param updateTask the full context for the cluster state update
+     *                   task
+     */
     public void submitStateUpdateTask(final String source, final ClusterStateUpdateTask updateTask) {
         submitStateUpdateTask(source, updateTask, updateTask, updateTask, updateTask);
     }
 
 
-    @Override
+    /**
+     * Submits a cluster state update task; submitted updates will be
+     * batched across the same instance of executor. The exact batching
+     * semantics depend on the underlying implementation but a rough
+     * guideline is that if the update task is submitted while there
+     * are pending update tasks for the same executor, these update
+     * tasks will all be executed on the executor in a single batch
+     *
+     * @param source   the source of the cluster state update task
+     * @param task     the state needed for the cluster state update task
+     * @param config   the cluster state update task configuration
+     * @param executor the cluster state update task executor; tasks
+     *                 that share the same executor will be executed
+     *                 batches on this executor
+     * @param listener callback after the cluster state update task
+     *                 completes
+     * @param <T>      the type of the cluster state update task state
+     */
     public <T> void submitStateUpdateTask(final String source, final T task,
                                           final ClusterStateTaskConfig config,
                                           final ClusterStateTaskExecutor<T> executor,
@@ -334,9 +373,9 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
     }
 
     private <T> void innerSubmitStateUpdateTask(final String source, final T task,
-                                           final ClusterStateTaskConfig config,
-                                           final ClusterStateTaskExecutor executor,
-                                           final SafeClusterStateTaskListener listener) {
+                                                final ClusterStateTaskConfig config,
+                                                final ClusterStateTaskExecutor executor,
+                                                final SafeClusterStateTaskListener listener) {
         if (!lifecycle.started()) {
             return;
         }
@@ -352,7 +391,8 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
                     if (updateTask.processed.getAndSet(true) == false) {
                         logger.debug("cluster state update task [{}] timed out after [{}]", source, config.timeout());
                         listener.onFailure(source, new ProcessClusterEventTimeoutException(config.timeout(), source));
-                    }}));
+                    }
+                }));
             } else {
                 updateTasksExecutor.execute(updateTask);
             }
@@ -365,7 +405,9 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
         }
     }
 
-    @Override
+    /**
+     * Returns the tasks that are pending.
+     */
     public List<PendingClusterTask> pendingTasks() {
         PrioritizedEsThreadPoolExecutor.Pending[] pendings = updateTasksExecutor.getPending();
         List<PendingClusterTask> pendingClusterTasks = new ArrayList<>(pendings.length);
@@ -386,29 +428,32 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
                 timeInQueue = 0;
             }
 
-            pendingClusterTasks.add(new PendingClusterTask(pending.insertionOrder, pending.priority, new Text(source), timeInQueue, pending.executing));
+            pendingClusterTasks.add(
+                    new PendingClusterTask(pending.insertionOrder, pending.priority, new Text(source), timeInQueue, pending.executing));
         }
         return pendingClusterTasks;
     }
 
-    @Override
+    /**
+     * Returns the number of currently pending tasks.
+     */
     public int numberOfPendingTasks() {
         return updateTasksExecutor.getNumberOfPendingTasks();
     }
 
-    @Override
+    /**
+     * Returns the maximum wait time for tasks in the queue
+     *
+     * @return A zero time value if the queue is empty, otherwise the time value oldest task waiting in the queue
+     */
     public TimeValue getMaxTaskWaitTime() {
         return updateTasksExecutor.getMaxTaskWaitTime();
     }
 
-    @Override
-    public TaskManager getTaskManager() {
-        return taskManager;
-    }
-
     /** asserts that the current thread is the cluster state update thread */
     public boolean assertClusterStateThread() {
-        assert Thread.currentThread().getName().contains(InternalClusterService.UPDATE_THREAD_NAME) : "not called from the cluster state update thread";
+        assert Thread.currentThread().getName().contains(ClusterService.UPDATE_THREAD_NAME) :
+                "not called from the cluster state update thread";
         return true;
     }
 
@@ -458,26 +503,27 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
             return;
         }
         ClusterStateTaskExecutor.BatchResult<T> batchResult;
-        long startTimeNS = System.nanoTime();
+        long startTimeNS = currentTimeInNanos();
         try {
             List<T> inputs = toExecute.stream().map(tUpdateTask -> tUpdateTask.task).collect(Collectors.toList());
             batchResult = executor.execute(previousClusterState, inputs);
         } catch (Throwable e) {
-            TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - startTimeNS)));
+            TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
             if (logger.isTraceEnabled()) {
-                StringBuilder sb = new StringBuilder("failed to execute cluster state update in ").append(executionTime).append(", state:\nversion [").append(previousClusterState.version()).append("], source [").append(source).append("]\n");
-                sb.append(previousClusterState.nodes().prettyPrint());
-                sb.append(previousClusterState.routingTable().prettyPrint());
-                sb.append(previousClusterState.getRoutingNodes().prettyPrint());
-                logger.trace(sb.toString(), e);
+                logger.trace("failed to execute cluster state update in [{}], state:\nversion [{}], source [{}]\n{}{}{}", e, executionTime,
+                        previousClusterState.version(), source, previousClusterState.nodes().prettyPrint(),
+                        previousClusterState.routingTable().prettyPrint(), previousClusterState.getRoutingNodes().prettyPrint());
             }
             warnAboutSlowTaskIfNeeded(executionTime, source);
-            batchResult = ClusterStateTaskExecutor.BatchResult.<T>builder().failures(toExecute.stream().map(updateTask -> updateTask.task)::iterator, e).build(previousClusterState);
+            batchResult = ClusterStateTaskExecutor.BatchResult.<T>builder()
+                    .failures(toExecute.stream().map(updateTask -> updateTask.task)::iterator, e)
+                    .build(previousClusterState);
         }
 
         assert batchResult.executionResults != null;
         assert batchResult.executionResults.size() == toExecute.size()
-            : String.format(Locale.ROOT, "expected [%d] task result%s but was [%d]", toExecute.size(), toExecute.size() == 1 ? "" : "s", batchResult.executionResults.size());
+                : String.format(Locale.ROOT, "expected [%d] task result%s but was [%d]", toExecute.size(),
+                toExecute.size() == 1 ? "" : "s", batchResult.executionResults.size());
         boolean assertsEnabled = false;
         assert (assertsEnabled = true);
         if (assertsEnabled) {
@@ -494,11 +540,11 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
             final ClusterStateTaskExecutor.TaskResult executionResult =
                     batchResult.executionResults.get(updateTask.task);
             executionResult.handle(
-                () -> proccessedListeners.add(updateTask),
-                ex -> {
-                    logger.debug("cluster state update task [{}] failed", ex, updateTask.source);
-                    updateTask.listener.onFailure(updateTask.source, ex);
-                }
+                    () -> proccessedListeners.add(updateTask),
+                    ex -> {
+                        logger.debug("cluster state update task [{}] failed", ex, updateTask.source);
+                        updateTask.listener.onFailure(updateTask.source, ex);
+                    }
             );
         }
 
@@ -510,8 +556,8 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
                 }
                 task.listener.clusterStateProcessed(task.source, previousClusterState, newClusterState);
             }
-            TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - startTimeNS)));
-            logger.debug("processing [{}]: took {} no change in cluster_state", source, executionTime);
+            TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
+            logger.debug("processing [{}]: took [{}] no change in cluster_state", source, executionTime);
             warnAboutSlowTaskIfNeeded(executionTime, source);
             return;
         }
@@ -522,7 +568,8 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
                 // only the master controls the version numbers
                 Builder builder = ClusterState.builder(newClusterState).incrementVersion();
                 if (previousClusterState.routingTable() != newClusterState.routingTable()) {
-                    builder.routingTable(RoutingTable.builder(newClusterState.routingTable()).version(newClusterState.routingTable().version() + 1).build());
+                    builder.routingTable(RoutingTable.builder(newClusterState.routingTable())
+                            .version(newClusterState.routingTable().version() + 1).build());
                 }
                 if (previousClusterState.metaData() != newClusterState.metaData()) {
                     builder.metaData(MetaData.builder(newClusterState.metaData()).version(newClusterState.metaData().version() + 1));
@@ -535,7 +582,8 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
                             ackedListener.onAckTimeout();
                         } else {
                             try {
-                                ackListeners.add(new AckCountDownListener(ackedListener, newClusterState.version(), newClusterState.nodes(), threadPool));
+                                ackListeners.add(new AckCountDownListener(ackedListener, newClusterState.version(), newClusterState.nodes(),
+                                        threadPool));
                             } catch (EsRejectedExecutionException ex) {
                                 if (logger.isDebugEnabled()) {
                                     logger.debug("Couldn't schedule timeout thread - node might be shutting down", ex);
@@ -552,9 +600,7 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
             newClusterState.status(ClusterState.ClusterStateStatus.BEING_APPLIED);
 
             if (logger.isTraceEnabled()) {
-                StringBuilder sb = new StringBuilder("cluster state updated, source [").append(source).append("]\n");
-                sb.append(newClusterState.prettyPrint());
-                logger.trace(sb.toString());
+                logger.trace("cluster state updated, source [{}]\n{}", source, newClusterState.prettyPrint());
             } else if (logger.isDebugEnabled()) {
                 logger.debug("cluster state updated, version [{}], source [{}]", newClusterState.version(), source);
             }
@@ -569,15 +615,7 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
                 }
             }
 
-            // TODO, do this in parallel (and wait)
-            for (DiscoveryNode node : nodesDelta.addedNodes()) {
-                try {
-                    transportService.connectToNode(node);
-                } catch (Throwable e) {
-                    // the fault detection will detect it as failed as well
-                    logger.warn("failed to connect to node [" + node + "]", e);
-                }
-            }
+            nodeConnectionsService.connectToAddedNodes(clusterChangedEvent);
 
             // if we are the master, publish the new state to all nodes
             // we publish here before we send a notification to all the listeners, since if it fails
@@ -613,13 +651,7 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
                 }
             }
 
-            for (DiscoveryNode node : nodesDelta.removedNodes()) {
-                try {
-                    transportService.disconnectFromNode(node);
-                } catch (Throwable e) {
-                    logger.warn("failed to disconnect to node [" + node + "]", e);
-                }
-            }
+            nodeConnectionsService.disconnectFromRemovedNodes(clusterChangedEvent);
 
             newClusterState.status(ClusterState.ClusterStateStatus.APPLIED);
 
@@ -650,21 +682,22 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
                 logger.error("exception thrown while notifying executor of new cluster state publication [{}]", e, source);
             }
 
-            TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - startTimeNS)));
-            logger.debug("processing [{}]: took {} done applying updated cluster_state (version: {}, uuid: {})", source, executionTime, newClusterState.version(), newClusterState.stateUUID());
+            TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
+            logger.debug("processing [{}]: took [{}] done applying updated cluster_state (version: {}, uuid: {})", source, executionTime,
+                    newClusterState.version(), newClusterState.stateUUID());
             warnAboutSlowTaskIfNeeded(executionTime, source);
         } catch (Throwable t) {
-            TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - startTimeNS)));
-            StringBuilder sb = new StringBuilder("failed to apply updated cluster state in ").append(executionTime).append(":\nversion [").append(newClusterState.version()).append("], uuid [").append(newClusterState.stateUUID()).append("], source [").append(source).append("]\n");
-            sb.append(newClusterState.nodes().prettyPrint());
-            sb.append(newClusterState.routingTable().prettyPrint());
-            sb.append(newClusterState.getRoutingNodes().prettyPrint());
-            logger.warn(sb.toString(), t);
+            TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
+            logger.warn("failed to apply updated cluster state in [{}]:\nversion [{}], uuid [{}], source [{}]\n{}", t, executionTime,
+                    newClusterState.version(), newClusterState.stateUUID(), source, newClusterState.prettyPrint());
             // TODO: do we want to call updateTask.onFailure here?
         }
 
     }
 
+    // this one is overridden in tests so we can control time
+    protected long currentTimeInNanos() {return System.nanoTime();}
+
     private static SafeClusterStateTaskListener safe(ClusterStateTaskListener listener, ESLogger logger) {
         if (listener instanceof AckedClusterStateTaskListener) {
             return new SafeAckedClusterStateTaskListener((AckedClusterStateTaskListener) listener, logger);
@@ -706,11 +739,12 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
                 listener.clusterStateProcessed(source, oldState, newState);
             } catch (Exception e) {
                 logger.error(
-                    "exception thrown by listener while notifying of cluster state processed from [{}], old cluster state:\n{}\nnew cluster state:\n{}",
-                    e,
-                    source,
-                    oldState.prettyPrint(),
-                    newState.prettyPrint());
+                        "exception thrown by listener while notifying of cluster state processed from [{}], old cluster state:\n" +
+                                "{}\nnew cluster state:\n{}",
+                        e,
+                        source,
+                        oldState.prettyPrint(),
+                        newState.prettyPrint());
             }
         }
     }
@@ -762,7 +796,8 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
         public final ClusterStateTaskListener listener;
         public final AtomicBoolean processed = new AtomicBoolean();
 
-        UpdateTask(String source, T task, ClusterStateTaskConfig config, ClusterStateTaskExecutor<T> executor, ClusterStateTaskListener listener) {
+        UpdateTask(String source, T task, ClusterStateTaskConfig config, ClusterStateTaskExecutor<T> executor,
+                   ClusterStateTaskListener listener) {
             super(config.priority(), source);
             this.task = task;
             this.config = config;
@@ -778,7 +813,8 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
 
     private void warnAboutSlowTaskIfNeeded(TimeValue executionTime, String source) {
         if (executionTime.getMillis() > slowTaskLoggingThreshold.getMillis()) {
-            logger.warn("cluster state update task [{}] took {} above the warn threshold of {}", source, executionTime, slowTaskLoggingThreshold);
+            logger.warn("cluster state update task [{}] took [{}] above the warn threshold of {}", source, executionTime,
+                    slowTaskLoggingThreshold);
         }
     }
 
@@ -810,64 +846,6 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
         }
     }
 
-    private class ReconnectToNodes implements Runnable {
-
-        private ConcurrentMap<DiscoveryNode, Integer> failureCount = ConcurrentCollections.newConcurrentMap();
-
-        @Override
-        public void run() {
-            // master node will check against all nodes if its alive with certain discoveries implementations,
-            // but we can't rely on that, so we check on it as well
-            for (DiscoveryNode node : clusterState.nodes()) {
-                if (lifecycle.stoppedOrClosed()) {
-                    return;
-                }
-                if (clusterState.nodes().nodeExists(node.id())) { // we double check existence of node since connectToNode might take time...
-                    if (!transportService.nodeConnected(node)) {
-                        try {
-                            transportService.connectToNode(node);
-                        } catch (Exception e) {
-                            if (lifecycle.stoppedOrClosed()) {
-                                return;
-                            }
-                            if (clusterState.nodes().nodeExists(node.id())) { // double check here as well, maybe its gone?
-                                Integer nodeFailureCount = failureCount.get(node);
-                                if (nodeFailureCount == null) {
-                                    nodeFailureCount = 1;
-                                } else {
-                                    nodeFailureCount = nodeFailureCount + 1;
-                                }
-                                // log every 6th failure
-                                if ((nodeFailureCount % 6) == 0) {
-                                    // reset the failure count...
-                                    nodeFailureCount = 0;
-                                    logger.warn("failed to reconnect to node {}", e, node);
-                                }
-                                failureCount.put(node, nodeFailureCount);
-                            }
-                        }
-                    }
-                }
-            }
-            // go over and remove failed nodes that have been removed
-            DiscoveryNodes nodes = clusterState.nodes();
-            for (Iterator<DiscoveryNode> failedNodesIt = failureCount.keySet().iterator(); failedNodesIt.hasNext(); ) {
-                DiscoveryNode failedNode = failedNodesIt.next();
-                if (!nodes.nodeExists(failedNode.id())) {
-                    failedNodesIt.remove();
-                }
-            }
-            if (lifecycle.started()) {
-                reconnectToNodes = threadPool.schedule(reconnectInterval, ThreadPool.Names.GENERIC, this);
-            }
-        }
-    }
-
-    public static String generateNodeId(Settings settings) {
-        Random random = Randomness.get(settings, NODE_ID_SEED_SETTING);
-        return Strings.randomBase64UUID(random);
-    }
-
     private static class LocalNodeMasterListeners implements ClusterStateListener {
 
         private final List<LocalNodeMasterListener> listeners = new CopyOnWriteArrayList<>();
@@ -971,7 +949,8 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
         private final Future<?> ackTimeoutCallback;
         private Throwable lastFailure;
 
-        AckCountDownListener(AckedClusterStateTaskListener ackedTaskListener, long clusterStateVersion, DiscoveryNodes nodes, ThreadPool threadPool) {
+        AckCountDownListener(AckedClusterStateTaskListener ackedTaskListener, long clusterStateVersion, DiscoveryNodes nodes,
+                             ThreadPool threadPool) {
             this.ackedTaskListener = ackedTaskListener;
             this.clusterStateVersion = clusterStateVersion;
             this.nodes = nodes;
@@ -1023,4 +1002,8 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
             }
         }
     }
+
+    public ClusterSettings getClusterSettings() {
+        return clusterSettings;
+    }
 }
diff --git a/core/src/main/java/org/apache/lucene/index/memory/ExtendedMemoryIndex.java b/core/src/main/java/org/elasticsearch/common/SuppressLoggerChecks.java
similarity index 63%
rename from core/src/main/java/org/apache/lucene/index/memory/ExtendedMemoryIndex.java
rename to core/src/main/java/org/elasticsearch/common/SuppressLoggerChecks.java
index aec1bc75519..c6f23f72f96 100644
--- a/core/src/main/java/org/apache/lucene/index/memory/ExtendedMemoryIndex.java
+++ b/core/src/main/java/org/elasticsearch/common/SuppressLoggerChecks.java
@@ -16,16 +16,18 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+package org.elasticsearch.common;
 
-package org.apache.lucene.index.memory;
 
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
 /**
- * This class overwrites {@link MemoryIndex} to make the reuse constructor visible.
+ * Annotation to suppress logging usage checks errors inside a whole class or a method.
  */
-public final class ExtendedMemoryIndex extends MemoryIndex {
-
-    public ExtendedMemoryIndex(boolean storeOffsets, boolean storePayloads, long maxReusedBytes) {
-        super(storeOffsets, storePayloads, maxReusedBytes);
-    }
-
+@Retention(RetentionPolicy.CLASS)
+@Target({ ElementType.CONSTRUCTOR, ElementType.METHOD, ElementType.TYPE })
+public @interface SuppressLoggerChecks {
+    String reason();
 }
diff --git a/core/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java b/core/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java
index 0a0be5eb163..e6c67dbe9ba 100644
--- a/core/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java
+++ b/core/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java
@@ -93,7 +93,7 @@ public class ChildMemoryCircuitBreaker implements CircuitBreaker {
         final String message = "[" + this.name + "] Data too large, data for [" +
                 fieldName + "] would be larger than limit of [" +
                 memoryBytesLimit + "/" + new ByteSizeValue(memoryBytesLimit) + "]";
-        logger.debug(message);
+        logger.debug("{}", message);
         throw new CircuitBreakingException(message,
                 bytesNeeded, this.memoryBytesLimit);
     }
diff --git a/core/src/main/java/org/elasticsearch/common/breaker/MemoryCircuitBreaker.java b/core/src/main/java/org/elasticsearch/common/breaker/MemoryCircuitBreaker.java
index b069456b5d4..3a5297b277b 100644
--- a/core/src/main/java/org/elasticsearch/common/breaker/MemoryCircuitBreaker.java
+++ b/core/src/main/java/org/elasticsearch/common/breaker/MemoryCircuitBreaker.java
@@ -81,7 +81,7 @@ public class MemoryCircuitBreaker implements CircuitBreaker {
         this.trippedCount.incrementAndGet();
         final String message = "Data too large, data for field [" + fieldName + "] would be larger than limit of [" +
                 memoryBytesLimit + "/" + new ByteSizeValue(memoryBytesLimit) + "]";
-        logger.debug(message);
+        logger.debug("{}", message);
         throw new CircuitBreakingException(message);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java
index a41b0388af7..16ce91dc38f 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java
@@ -445,5 +445,10 @@ public class PagedBytesReference implements BytesReference {
             // do nothing
         }
 
+        @Override
+        public int available() throws IOException {
+            return length - pos;
+        }
+
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/cli/CheckFileCommand.java b/core/src/main/java/org/elasticsearch/common/cli/CheckFileCommand.java
deleted file mode 100644
index e2fcbe89df8..00000000000
--- a/core/src/main/java/org/elasticsearch/common/cli/CheckFileCommand.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.cli;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.attribute.PosixFileAttributeView;
-import java.nio.file.attribute.PosixFileAttributes;
-import java.nio.file.attribute.PosixFilePermission;
-import java.nio.file.attribute.PosixFilePermissions;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * A helper command that checks if configured paths have been changed when running a CLI command.
- * It is only executed in case of specified paths by the command and if the paths underlying filesystem
- * supports posix permissions.
- *
- * If this is the case, a warn message is issued whenever an owner, a group or the file permissions is changed by
- * the command being executed and not configured back to its prior state, which should be the task of the command
- * being executed.
- *
- */
-public abstract class CheckFileCommand extends CliTool.Command {
-
-    public CheckFileCommand(Terminal terminal) {
-        super(terminal);
-    }
-
-    /**
-     * abstract method, which should implement the same logic as CliTool.Command.execute(), but is wrapped
-     */
-    public abstract CliTool.ExitStatus doExecute(Settings settings, Environment env) throws Exception;
-
-    /**
-     * Returns the array of paths, that should be checked if the permissions, user or groups have changed
-     * before and after execution of the command
-     *
-     */
-    protected abstract Path[] pathsForPermissionsCheck(Settings settings, Environment env) throws Exception;
-
-    @Override
-    public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
-        Path[] paths = pathsForPermissionsCheck(settings, env);
-
-        if (paths == null || paths.length == 0) {
-            return doExecute(settings, env);
-        }
-
-        Map<Path, Set<PosixFilePermission>> permissions = new HashMap<>(paths.length);
-        Map<Path, String> owners = new HashMap<>(paths.length);
-        Map<Path, String> groups = new HashMap<>(paths.length);
-
-        if (paths != null && paths.length > 0) {
-            for (Path path : paths) {
-                try {
-                    boolean supportsPosixPermissions = Environment.getFileStore(path).supportsFileAttributeView(PosixFileAttributeView.class);
-                    if (supportsPosixPermissions) {
-                        PosixFileAttributes attributes = Files.readAttributes(path, PosixFileAttributes.class);
-                        permissions.put(path, attributes.permissions());
-                        owners.put(path, attributes.owner().getName());
-                        groups.put(path, attributes.group().getName());
-                    }
-                } catch (IOException e) {
-                    // silently swallow if not supported, no need to log things
-                }
-            }
-        }
-
-        CliTool.ExitStatus status = doExecute(settings, env);
-
-        // check if permissions differ
-        for (Map.Entry<Path, Set<PosixFilePermission>> entry : permissions.entrySet()) {
-            if (!Files.exists(entry.getKey())) {
-                continue;
-            }
-
-            Set<PosixFilePermission> permissionsBeforeWrite = entry.getValue();
-            Set<PosixFilePermission> permissionsAfterWrite = Files.getPosixFilePermissions(entry.getKey());
-            if (!permissionsBeforeWrite.equals(permissionsAfterWrite)) {
-                terminal.println(Terminal.Verbosity.SILENT, "WARNING: The file permissions of [" + entry.getKey() + "] have changed "
-                        + "from [" + PosixFilePermissions.toString(permissionsBeforeWrite) + "] "
-                        + "to [" + PosixFilePermissions.toString(permissionsAfterWrite) + "]");
-                terminal.println(Terminal.Verbosity.SILENT, "Please ensure that the user account running Elasticsearch has read access to this file!");
-            }
-        }
-
-        // check if owner differs
-        for (Map.Entry<Path, String> entry : owners.entrySet()) {
-            if (!Files.exists(entry.getKey())) {
-                continue;
-            }
-
-            String ownerBeforeWrite = entry.getValue();
-            String ownerAfterWrite = Files.getOwner(entry.getKey()).getName();
-            if (!ownerAfterWrite.equals(ownerBeforeWrite)) {
-                terminal.println(Terminal.Verbosity.SILENT, "WARNING: Owner of file [" + entry.getKey() + "] used to be [" + ownerBeforeWrite + "], but now is [" + ownerAfterWrite + "]");
-            }
-        }
-
-        // check if group differs
-        for (Map.Entry<Path, String> entry : groups.entrySet()) {
-            if (!Files.exists(entry.getKey())) {
-                continue;
-            }
-
-            String groupBeforeWrite = entry.getValue();
-            String groupAfterWrite = Files.readAttributes(entry.getKey(), PosixFileAttributes.class).group().getName();
-            if (!groupAfterWrite.equals(groupBeforeWrite)) {
-                terminal.println(Terminal.Verbosity.SILENT, "WARNING: Group of file [" + entry.getKey() + "] used to be [" + groupBeforeWrite + "], but now is [" + groupAfterWrite + "]");
-            }
-        }
-
-        return status;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/common/cli/CliTool.java b/core/src/main/java/org/elasticsearch/common/cli/CliTool.java
deleted file mode 100644
index 2ea01f45068..00000000000
--- a/core/src/main/java/org/elasticsearch/common/cli/CliTool.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.cli;
-
-import org.apache.commons.cli.AlreadySelectedException;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.DefaultParser;
-import org.apache.commons.cli.MissingArgumentException;
-import org.apache.commons.cli.MissingOptionException;
-import org.apache.commons.cli.UnrecognizedOptionException;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
-
-import java.util.Locale;
-
-import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
-
-/**
- * A base class for command-line interface tool.
- *
- * Two modes are supported:
- *
- * - Single command mode. The tool exposes a single command that can potentially accept arguments (eg. CLI options).
- * - Multi command mode. The tool support multiple commands, each for different tasks, each potentially accepts arguments.
- *
- * In a multi-command mode. The first argument must be the command name. For example, the plugin manager
- * can be seen as a multi-command tool with two possible commands: install and uninstall
- *
- * The tool is configured using a {@link CliToolConfig} which encapsulates the tool's commands and their
- * potential options. The tool also comes with out of the box simple help support (the -h/--help option is
- * automatically handled) where the help text is configured in a dedicated *.help files located in the same package
- * as the tool.
- */
-public abstract class CliTool {
-
-    // based on sysexits.h
-    public enum ExitStatus {
-        OK(0),
-        OK_AND_EXIT(0),
-        USAGE(64),          /* command line usage error */
-        DATA_ERROR(65),     /* data format error */
-        NO_INPUT(66),       /* cannot open input */
-        NO_USER(67),        /* addressee unknown */
-        NO_HOST(68),        /* host name unknown */
-        UNAVAILABLE(69),    /* service unavailable */
-        CODE_ERROR(70),     /* internal software error */
-        CANT_CREATE(73),    /* can't create (user) output file */
-        IO_ERROR(74),       /* input/output error */
-        TEMP_FAILURE(75),   /* temp failure; user is invited to retry */
-        PROTOCOL(76),       /* remote error in protocol */
-        NOPERM(77),         /* permission denied */
-        CONFIG(78);          /* configuration error */
-
-        final int status;
-
-        ExitStatus(int status) {
-            this.status = status;
-        }
-
-        public int status() {
-            return status;
-        }
-    }
-
-    protected final Terminal terminal;
-    protected final Environment env;
-    protected final Settings settings;
-
-    private final CliToolConfig config;
-
-    protected CliTool(CliToolConfig config) {
-        this(config, Terminal.DEFAULT);
-    }
-
-    protected CliTool(CliToolConfig config, Terminal terminal) {
-        if (config.cmds().size() == 0) {
-            throw new IllegalArgumentException("At least one command must be configured");
-        }
-        this.config = config;
-        this.terminal = terminal;
-        env = InternalSettingsPreparer.prepareEnvironment(EMPTY_SETTINGS, terminal);
-        settings = env.settings();
-    }
-
-    public final ExitStatus execute(String... args) throws Exception {
-
-        // first lets see if the user requests tool help. We're doing it only if
-        // this is a multi-command tool. If it's a single command tool, the -h/--help
-        // option will be taken care of on the command level
-        if (!config.isSingle() && args.length > 0 && (args[0].equals("-h") || args[0].equals("--help"))) {
-            config.printUsage(terminal);
-            return ExitStatus.OK_AND_EXIT;
-        }
-
-        CliToolConfig.Cmd cmd;
-        if (config.isSingle()) {
-            cmd = config.single();
-        } else {
-
-            if (args.length == 0) {
-                terminal.println(Terminal.Verbosity.SILENT, "ERROR: command not specified");
-                config.printUsage(terminal);
-                return ExitStatus.USAGE;
-            }
-
-            String cmdName = args[0];
-            cmd = config.cmd(cmdName);
-            if (cmd == null) {
-                terminal.println(Terminal.Verbosity.SILENT, "ERROR: unknown command [" + cmdName + "]. Use [-h] option to list available commands");
-                return ExitStatus.USAGE;
-            }
-
-            // we now remove the command name from the args
-            if (args.length == 1) {
-                args = new String[0];
-            } else {
-                String[] cmdArgs = new String[args.length - 1];
-                System.arraycopy(args, 1, cmdArgs, 0, cmdArgs.length);
-                args = cmdArgs;
-            }
-        }
-
-        try {
-            return parse(cmd, args).execute(settings, env);
-        } catch (UserError error) {
-            terminal.println(Terminal.Verbosity.SILENT, "ERROR: " + error.getMessage());
-            return error.exitStatus;
-        }
-    }
-
-    public Command parse(String cmdName, String[] args) throws Exception {
-        CliToolConfig.Cmd cmd = config.cmd(cmdName);
-        return parse(cmd, args);
-    }
-
-    public Command parse(CliToolConfig.Cmd cmd, String[] args) throws Exception {
-        CommandLineParser parser = new DefaultParser();
-        CommandLine cli = parser.parse(CliToolConfig.OptionsSource.HELP.options(), args, true);
-        if (cli.hasOption("h")) {
-            return helpCmd(cmd);
-        }
-        try {
-            cli = parser.parse(cmd.options(), args, cmd.isStopAtNonOption());
-        } catch (AlreadySelectedException|MissingArgumentException|MissingOptionException|UnrecognizedOptionException e) {
-            // intentionally drop the stack trace here as these are really user errors,
-            // the stack trace into cli parsing lib is not important
-            throw new UserError(ExitStatus.USAGE, e.toString());
-        }
-
-        if (cli.hasOption("v")) {
-            terminal.setVerbosity(Terminal.Verbosity.VERBOSE);
-        } else if (cli.hasOption("s")) {
-            terminal.setVerbosity(Terminal.Verbosity.SILENT);
-        } else {
-            terminal.setVerbosity(Terminal.Verbosity.NORMAL);
-        }
-        return parse(cmd.name(), cli);
-    }
-
-    protected Command.Help helpCmd(CliToolConfig.Cmd cmd) {
-        return new Command.Help(cmd, terminal);
-    }
-
-    protected static Command.Exit exitCmd(ExitStatus status) {
-        return new Command.Exit(null, status, null);
-    }
-
-    protected static Command.Exit exitCmd(ExitStatus status, Terminal terminal, String msg, Object... args) {
-        return new Command.Exit(String.format(Locale.ROOT, msg, args), status, terminal);
-    }
-
-    protected abstract Command parse(String cmdName, CommandLine cli) throws Exception;
-
-    public static abstract class Command {
-
-        protected final Terminal terminal;
-
-        protected Command(Terminal terminal) {
-            this.terminal = terminal;
-        }
-
-        public abstract ExitStatus execute(Settings settings, Environment env) throws Exception;
-
-        public static class Help extends Command {
-
-            private final CliToolConfig.Cmd cmd;
-
-            private Help(CliToolConfig.Cmd cmd, Terminal terminal) {
-                super(terminal);
-                this.cmd = cmd;
-            }
-
-            @Override
-            public ExitStatus execute(Settings settings, Environment env) throws Exception {
-                cmd.printUsage(terminal);
-                return ExitStatus.OK_AND_EXIT;
-            }
-        }
-
-        public static class Exit extends Command {
-            private final String msg;
-            private final ExitStatus status;
-
-            private Exit(String msg, ExitStatus status, Terminal terminal) {
-                super(terminal);
-                this.msg = msg;
-                this.status = status;
-            }
-
-            @Override
-            public ExitStatus execute(Settings settings, Environment env) throws Exception {
-                if (msg != null) {
-                    if (status != ExitStatus.OK) {
-                        terminal.println(Terminal.Verbosity.SILENT, "ERROR: " + msg);
-                    } else {
-                        terminal.println(msg);
-                    }
-                }
-                return status;
-            }
-
-            public ExitStatus status() {
-                return status;
-            }
-        }
-    }
-
-
-
-}
-
diff --git a/core/src/main/java/org/elasticsearch/common/cli/CliToolConfig.java b/core/src/main/java/org/elasticsearch/common/cli/CliToolConfig.java
deleted file mode 100644
index d0ba897b33d..00000000000
--- a/core/src/main/java/org/elasticsearch/common/cli/CliToolConfig.java
+++ /dev/null
@@ -1,302 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.cli;
-
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionGroup;
-import org.apache.commons.cli.Options;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- *
- */
-public class CliToolConfig {
-
-    public static Builder config(String name, Class<? extends CliTool> toolType) {
-        return new Builder(name, toolType);
-    }
-
-    private final Class<? extends CliTool> toolType;
-    private final String name;
-    private final Map<String, Cmd> cmds;
-
-    private static final HelpPrinter helpPrinter = new HelpPrinter();
-
-    private CliToolConfig(String name, Class<? extends CliTool> toolType, Cmd[] cmds) {
-        this.name = name;
-        this.toolType = toolType;
-        final Map<String, Cmd> cmdsMapping = new HashMap<>();
-        for (int i = 0; i < cmds.length; i++) {
-            cmdsMapping.put(cmds[i].name, cmds[i]);
-        }
-        this.cmds = Collections.unmodifiableMap(cmdsMapping);
-    }
-
-    public boolean isSingle() {
-        return cmds.size() == 1;
-    }
-
-    public Cmd single() {
-        assert isSingle() : "Requesting single command on a multi-command tool";
-        return cmds.values().iterator().next();
-    }
-
-    public Class<? extends CliTool> toolType() {
-        return toolType;
-    }
-
-    public String name() {
-        return name;
-    }
-
-    public Collection<Cmd> cmds() {
-        return cmds.values();
-    }
-
-    public Cmd cmd(String name) {
-        return cmds.get(name);
-    }
-
-    public void printUsage(Terminal terminal) {
-        helpPrinter.print(this, terminal);
-    }
-
-    public static class Builder {
-
-        public static Cmd.Builder cmd(String name, Class<? extends CliTool.Command> cmdType) {
-            return new Cmd.Builder(name, cmdType);
-        }
-
-        public static OptionBuilder option(String shortName, String longName) {
-            return new OptionBuilder(shortName, longName);
-        }
-
-        public static Option.Builder optionBuilder(String shortName, String longName) {
-            return Option.builder(shortName).argName(longName).longOpt(longName);
-        }
-
-        public static OptionGroupBuilder optionGroup(boolean required) {
-            return new OptionGroupBuilder(required);
-        }
-
-        private final Class<? extends CliTool> toolType;
-        private final String name;
-        private Cmd[] cmds;
-
-        private Builder(String name, Class<? extends CliTool> toolType) {
-            this.name = name;
-            this.toolType = toolType;
-        }
-
-        public Builder cmds(Cmd.Builder... cmds) {
-            this.cmds = new Cmd[cmds.length];
-            for (int i = 0; i < cmds.length; i++) {
-                this.cmds[i] = cmds[i].build();
-                this.cmds[i].toolName = name;
-            }
-            return this;
-        }
-
-        public Builder cmds(Cmd... cmds) {
-            for (int i = 0; i < cmds.length; i++) {
-                cmds[i].toolName = name;
-            }
-            this.cmds = cmds;
-            return this;
-        }
-
-        public CliToolConfig build() {
-            return new CliToolConfig(name, toolType, cmds);
-        }
-    }
-
-    public static class Cmd {
-
-        private String toolName;
-        private final String name;
-        private final Class<? extends CliTool.Command> cmdType;
-        private final Options options;
-        private final boolean stopAtNonOption;
-
-        private Cmd(String name, Class<? extends CliTool.Command> cmdType, Options options, boolean stopAtNonOption) {
-            this.name = name;
-            this.cmdType = cmdType;
-            this.options = options;
-            this.stopAtNonOption = stopAtNonOption;
-            OptionsSource.VERBOSITY.populate(options);
-        }
-
-        public Class<? extends CliTool.Command> cmdType() {
-            return cmdType;
-        }
-
-        public String name() {
-            return name;
-        }
-
-        public Options options() {
-            return options;
-        }
-
-        public boolean isStopAtNonOption() {
-            return stopAtNonOption;
-        }
-
-        public void printUsage(Terminal terminal) {
-            helpPrinter.print(toolName, this, terminal);
-        }
-
-        public static class Builder {
-
-            private final String name;
-            private final Class<? extends CliTool.Command> cmdType;
-            private Options options = new Options();
-            private boolean stopAtNonOption = false;
-
-            private Builder(String name, Class<? extends CliTool.Command> cmdType) {
-                this.name = name;
-                this.cmdType = cmdType;
-            }
-
-            public Builder options(OptionBuilder... optionBuilder) {
-                for (int i = 0; i < optionBuilder.length; i++) {
-                    options.addOption(optionBuilder[i].build());
-                }
-                return this;
-            }
-
-            public Builder options(Option.Builder... optionBuilders) {
-                for (int i = 0; i < optionBuilders.length; i++) {
-                    options.addOption(optionBuilders[i].build());
-                }
-                return this;
-            }
-
-            public Builder optionGroups(OptionGroupBuilder... optionGroupBuilders) {
-                for (OptionGroupBuilder builder : optionGroupBuilders) {
-                    options.addOptionGroup(builder.build());
-                }
-                return this;
-            }
-
-            /**
-              * @param stopAtNonOption if <tt>true</tt> an unrecognized argument stops
-              *     the parsing and the remaining arguments are added to the
-              *     args list. If <tt>false</tt> an unrecognized
-              *     argument triggers a ParseException.
-              */
-            public Builder stopAtNonOption(boolean stopAtNonOption) {
-                this.stopAtNonOption = stopAtNonOption;
-                return this;
-            }
-
-            public Cmd build() {
-                return new Cmd(name, cmdType, options, stopAtNonOption);
-            }
-        }
-    }
-
-    public static class OptionBuilder {
-
-        private final Option option;
-
-        private OptionBuilder(String shortName, String longName) {
-            option = new Option(shortName, "");
-            option.setLongOpt(longName);
-            option.setArgName(longName);
-        }
-
-        public OptionBuilder required(boolean required) {
-            option.setRequired(required);
-            return this;
-        }
-
-        public OptionBuilder hasArg(boolean optional) {
-            option.setOptionalArg(optional);
-            option.setArgs(1);
-            return this;
-        }
-
-        public Option build() {
-            return option;
-        }
-    }
-
-    public static class OptionGroupBuilder {
-
-        private OptionGroup group;
-
-        private OptionGroupBuilder(boolean required) {
-            group = new OptionGroup();
-            group.setRequired(required);
-        }
-
-        public OptionGroupBuilder options(OptionBuilder... optionBuilders) {
-            for (OptionBuilder builder : optionBuilders) {
-                group.addOption(builder.build());
-            }
-            return this;
-        }
-
-        public OptionGroup build() {
-            return group;
-        }
-
-    }
-
-    static abstract class OptionsSource {
-
-        static final OptionsSource HELP = new OptionsSource() {
-
-            @Override
-            void populate(Options options) {
-                options.addOption(new OptionBuilder("h", "help").required(false).build());
-            }
-        };
-
-        static final OptionsSource VERBOSITY = new OptionsSource() {
-            @Override
-            void populate(Options options) {
-                OptionGroup verbosityGroup = new OptionGroup();
-                verbosityGroup.setRequired(false);
-                verbosityGroup.addOption(new OptionBuilder("s", "silent").required(false).build());
-                verbosityGroup.addOption(new OptionBuilder("v", "verbose").required(false).build());
-                options.addOptionGroup(verbosityGroup);
-            }
-        };
-
-        private Options options;
-
-        Options options() {
-            if (options == null) {
-                options = new Options();
-                populate(options);
-            }
-            return options;
-        }
-
-        abstract void populate(Options options);
-
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/common/cli/HelpPrinter.java b/core/src/main/java/org/elasticsearch/common/cli/HelpPrinter.java
deleted file mode 100644
index ada6cc33a19..00000000000
--- a/core/src/main/java/org/elasticsearch/common/cli/HelpPrinter.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.cli;
-
-import org.elasticsearch.common.io.Streams;
-import org.elasticsearch.common.util.Callback;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- *
- */
-public class HelpPrinter {
-
-    private static final String HELP_FILE_EXT = ".help";
-
-    public void print(CliToolConfig config, Terminal terminal) {
-        print(config.toolType(), config.name(), terminal);
-    }
-
-    public void print(String toolName, CliToolConfig.Cmd cmd, Terminal terminal) {
-        print(cmd.cmdType(), toolName + "-" + cmd.name(), terminal);
-    }
-
-    private static void print(Class clazz, String name, final Terminal terminal) {
-        terminal.println(Terminal.Verbosity.SILENT, "");
-        try (InputStream input = clazz.getResourceAsStream(name + HELP_FILE_EXT)) {
-            Streams.readAllLines(input, new Callback<String>() {
-                @Override
-                public void handle(String line) {
-                    terminal.println(Terminal.Verbosity.SILENT, line);
-                }
-            });
-        } catch (IOException ioe) {
-            throw new RuntimeException(ioe);
-        }
-        terminal.println(Terminal.Verbosity.SILENT, "");
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/CircleBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/CircleBuilder.java
index 97ef6561c9b..658d8ed84c1 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/CircleBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/CircleBuilder.java
@@ -36,8 +36,6 @@ public class CircleBuilder extends ShapeBuilder {
     public static final String FIELD_RADIUS = "radius";
     public static final GeoShapeType TYPE = GeoShapeType.CIRCLE;
 
-    public static final CircleBuilder PROTOTYPE = new CircleBuilder();
-
     private DistanceUnit unit = DistanceUnit.DEFAULT;
     private double radius;
     private Coordinate center;
@@ -50,6 +48,21 @@ public class CircleBuilder extends ShapeBuilder {
         this.center = ZERO_ZERO;
     }
 
+    /**
+     * Read from a stream.
+     */
+    public CircleBuilder(StreamInput in) throws IOException {
+        center(readFromStream(in));
+        radius(in.readDouble(), DistanceUnit.readFromStream(in));;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        writeCoordinateTo(center, out);
+        out.writeDouble(radius);
+        unit.writeTo(out);
+    }
+
     /**
      * Set the center of the circle
      *
@@ -170,18 +183,4 @@ public class CircleBuilder extends ShapeBuilder {
                 Objects.equals(radius, other.radius) &&
                 Objects.equals(unit.ordinal(), other.unit.ordinal());
     }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        writeCoordinateTo(center, out);
-        out.writeDouble(radius);
-        DistanceUnit.writeDistanceUnit(out, unit);
-    }
-
-    @Override
-    public CircleBuilder readFrom(StreamInput in) throws IOException {
-        return new CircleBuilder()
-                    .center(readCoordinateFrom(in))
-                    .radius(in.readDouble(), DistanceUnit.readDistanceUnit(in));
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/CoordinateCollection.java b/core/src/main/java/org/elasticsearch/common/geo/builders/CoordinateCollection.java
index 72ac7be8114..b6b9df45d04 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/CoordinateCollection.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/CoordinateCollection.java
@@ -21,9 +21,12 @@ package org.elasticsearch.common.geo.builders;
 
 import com.vividsolutions.jts.geom.Coordinate;
 
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
@@ -48,6 +51,25 @@ public abstract class CoordinateCollection<E extends CoordinateCollection<E>> ex
         this.coordinates = coordinates;
     }
 
+    /**
+     * Read from a stream.
+     */
+    protected CoordinateCollection(StreamInput in) throws IOException {
+        int size = in.readVInt();
+        coordinates = new ArrayList<>(size);
+        for (int i=0; i < size; i++) {
+            coordinates.add(readFromStream(in));
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(coordinates.size());
+        for (Coordinate point : coordinates) {
+            writeCoordinateTo(point, out);
+        }
+    }
+
     @SuppressWarnings("unchecked")
     private E thisRef() {
         return (E)this;
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/EnvelopeBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/EnvelopeBuilder.java
index ab997387ea1..5b80ceeeeea 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/EnvelopeBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/EnvelopeBuilder.java
@@ -33,11 +33,12 @@ public class EnvelopeBuilder extends ShapeBuilder {
 
     public static final GeoShapeType TYPE = GeoShapeType.ENVELOPE;
 
-    public static final EnvelopeBuilder PROTOTYPE = new EnvelopeBuilder(new Coordinate(-1.0, 1.0), new Coordinate(1.0, -1.0));
-
-    private Coordinate topLeft;
-    private Coordinate bottomRight;
+    private final Coordinate topLeft;
+    private final Coordinate bottomRight;
 
+    /**
+     * Build an envelope from the top left and bottom right coordinates.
+     */
     public EnvelopeBuilder(Coordinate topLeft, Coordinate bottomRight) {
         Objects.requireNonNull(topLeft, "topLeft of envelope cannot be null");
         Objects.requireNonNull(bottomRight, "bottomRight of envelope cannot be null");
@@ -45,6 +46,20 @@ public class EnvelopeBuilder extends ShapeBuilder {
         this.bottomRight = bottomRight;
     }
 
+    /**
+     * Read from a stream.
+     */
+    public EnvelopeBuilder(StreamInput in) throws IOException {
+        topLeft = readFromStream(in);
+        bottomRight = readFromStream(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        writeCoordinateTo(topLeft, out);
+        writeCoordinateTo(bottomRight, out);
+    }
+
     public Coordinate topLeft() {
         return this.topLeft;
     }
@@ -91,15 +106,4 @@ public class EnvelopeBuilder extends ShapeBuilder {
         return Objects.equals(topLeft, other.topLeft) &&
                 Objects.equals(bottomRight, other.bottomRight);
     }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        writeCoordinateTo(topLeft, out);
-        writeCoordinateTo(bottomRight, out);
-    }
-
-    @Override
-    public EnvelopeBuilder readFrom(StreamInput in) throws IOException {
-        return new EnvelopeBuilder(readCoordinateFrom(in), readCoordinateFrom(in));
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java
index d21f47cf053..b8559fcb48f 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java
@@ -36,9 +36,34 @@ public class GeometryCollectionBuilder extends ShapeBuilder {
 
     public static final GeoShapeType TYPE = GeoShapeType.GEOMETRYCOLLECTION;
 
-    public static final GeometryCollectionBuilder PROTOTYPE = new GeometryCollectionBuilder();
+    /**
+     * List of shapes. Package scope for testing.
+     */
+    final List<ShapeBuilder> shapes = new ArrayList<>();
 
-    protected final ArrayList<ShapeBuilder> shapes = new ArrayList<>();
+    /**
+     * Build and empty GeometryCollectionBuilder.
+     */
+    public GeometryCollectionBuilder() {
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public GeometryCollectionBuilder(StreamInput in) throws IOException {
+        int shapes = in.readVInt();
+        for (int i = 0; i < shapes; i++) {
+            shape(in.readShape());
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(shapes.size());
+        for (ShapeBuilder shape : shapes) {
+            out.writeShape(shape);
+        }
+    }
 
     public GeometryCollectionBuilder shape(ShapeBuilder shape) {
         this.shapes.add(shape);
@@ -146,23 +171,4 @@ public class GeometryCollectionBuilder extends ShapeBuilder {
         GeometryCollectionBuilder other = (GeometryCollectionBuilder) obj;
         return Objects.equals(shapes, other.shapes);
     }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(shapes.size());
-        for (ShapeBuilder shape : shapes) {
-            out.writeShape(shape);
-        }
-    }
-
-    @Override
-    public GeometryCollectionBuilder readFrom(StreamInput in) throws IOException {
-        GeometryCollectionBuilder geometryCollectionBuilder = new GeometryCollectionBuilder();
-        int shapes = in.readVInt();
-        for (int i = 0; i < shapes; i++) {
-            geometryCollectionBuilder.shape(in.readShape());
-        }
-        return geometryCollectionBuilder;
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/LineStringBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/LineStringBuilder.java
index cbc9002c785..e79578d9ab2 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/LineStringBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/LineStringBuilder.java
@@ -19,15 +19,14 @@
 
 package org.elasticsearch.common.geo.builders;
 
-import org.locationtech.spatial4j.shape.Shape;
 import com.vividsolutions.jts.geom.Coordinate;
 import com.vividsolutions.jts.geom.Geometry;
 import com.vividsolutions.jts.geom.GeometryFactory;
 import com.vividsolutions.jts.geom.LineString;
 
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.locationtech.spatial4j.shape.Shape;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -36,6 +35,7 @@ import java.util.List;
 import java.util.Objects;
 
 public class LineStringBuilder extends CoordinateCollection<LineStringBuilder> {
+    public static final GeoShapeType TYPE = GeoShapeType.LINESTRING;
 
     /**
      * Construct a new LineString.
@@ -55,9 +55,12 @@ public class LineStringBuilder extends CoordinateCollection<LineStringBuilder> {
         this(coordinates.build());
     }
 
-    public static final GeoShapeType TYPE = GeoShapeType.LINESTRING;
-
-    public static final LineStringBuilder PROTOTYPE = new LineStringBuilder(new CoordinatesBuilder().coordinate(0.0, 0.0).coordinate(1.0, 1.0));
+    /**
+     * Read from a stream.
+     */
+    public LineStringBuilder(StreamInput in) throws IOException {
+        super(in);
+    }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
@@ -182,23 +185,4 @@ public class LineStringBuilder extends CoordinateCollection<LineStringBuilder> {
         LineStringBuilder other = (LineStringBuilder) obj;
         return Objects.equals(coordinates, other.coordinates);
     }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(coordinates.size());
-        for (Coordinate point : coordinates) {
-            writeCoordinateTo(point, out);
-        }
-    }
-
-    @Override
-    public LineStringBuilder readFrom(StreamInput in) throws IOException {
-        CoordinatesBuilder coordinates = new CoordinatesBuilder();
-        int size = in.readVInt();
-        for (int i=0; i < size; i++) {
-            coordinates.coordinate(readCoordinateFrom(in));
-        }
-        LineStringBuilder lineStringBuilder = new LineStringBuilder(coordinates);
-        return lineStringBuilder;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java
index 51f4fd232c5..04e25862c8b 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java
@@ -37,10 +37,29 @@ public class MultiLineStringBuilder extends ShapeBuilder {
 
     public static final GeoShapeType TYPE = GeoShapeType.MULTILINESTRING;
 
-    public static final MultiLineStringBuilder PROTOTYPE = new MultiLineStringBuilder();
-
     private final ArrayList<LineStringBuilder> lines = new ArrayList<>();
 
+    public MultiLineStringBuilder() {
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public MultiLineStringBuilder(StreamInput in) throws IOException {
+        int size = in.readVInt();
+        for (int i = 0; i < size; i++) {
+            linestring(new LineStringBuilder(in));
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(lines.size());
+        for (LineStringBuilder line : lines) {
+            line.writeTo(out);
+        }
+    }
+
     public MultiLineStringBuilder linestring(LineStringBuilder line) {
         this.lines.add(line);
         return this;
@@ -114,22 +133,4 @@ public class MultiLineStringBuilder extends ShapeBuilder {
         MultiLineStringBuilder other = (MultiLineStringBuilder) obj;
         return Objects.equals(lines, other.lines);
     }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(lines.size());
-        for (LineStringBuilder line : lines) {
-            line.writeTo(out);
-        }
-    }
-
-    @Override
-    public MultiLineStringBuilder readFrom(StreamInput in) throws IOException {
-        MultiLineStringBuilder multiLineStringBuilder = new MultiLineStringBuilder();
-        int size = in.readVInt();
-        for (int i = 0; i < size; i++) {
-            multiLineStringBuilder.linestring(LineStringBuilder.PROTOTYPE.readFrom(in));
-        }
-        return multiLineStringBuilder;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java
index b8f2c8137ef..f8a06244362 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPointBuilder.java
@@ -19,14 +19,13 @@
 
 package org.elasticsearch.common.geo.builders;
 
-import org.locationtech.spatial4j.shape.Point;
-import org.locationtech.spatial4j.shape.Shape;
 import com.vividsolutions.jts.geom.Coordinate;
 
 import org.elasticsearch.common.geo.XShapeCollection;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.locationtech.spatial4j.shape.Point;
+import org.locationtech.spatial4j.shape.Shape;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -37,8 +36,6 @@ public class MultiPointBuilder extends CoordinateCollection<MultiPointBuilder> {
 
     public static final GeoShapeType TYPE = GeoShapeType.MULTIPOINT;
 
-    public static final MultiPointBuilder PROTOTYPE = new MultiPointBuilder(new CoordinatesBuilder().coordinate(0.0, 0.0).build());
-
     /**
      * Create a new {@link MultiPointBuilder}.
      * @param coordinates needs at least two coordinates to be valid, otherwise will throw an exception
@@ -47,6 +44,13 @@ public class MultiPointBuilder extends CoordinateCollection<MultiPointBuilder> {
         super(coordinates);
     }
 
+    /**
+     * Read from a stream.
+     */
+    public MultiPointBuilder(StreamInput in) throws IOException {
+        super(in);
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -91,24 +95,4 @@ public class MultiPointBuilder extends CoordinateCollection<MultiPointBuilder> {
         MultiPointBuilder other = (MultiPointBuilder) obj;
         return Objects.equals(coordinates, other.coordinates);
     }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(coordinates.size());
-        for (Coordinate point : coordinates) {
-            writeCoordinateTo(point, out);
-        }
-    }
-
-    @Override
-    public MultiPointBuilder readFrom(StreamInput in) throws IOException {
-        int size = in.readVInt();
-        List<Coordinate> points = new ArrayList<Coordinate>(size);
-        for (int i=0; i < size; i++) {
-            points.add(readCoordinateFrom(in));
-        }
-        MultiPointBuilder multiPointBuilder = new MultiPointBuilder(points);
-
-        return multiPointBuilder;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java
index 6ee679b7308..f5e5bca5051 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java
@@ -36,20 +36,45 @@ import java.util.Objects;
 public class MultiPolygonBuilder extends ShapeBuilder {
 
     public static final GeoShapeType TYPE = GeoShapeType.MULTIPOLYGON;
-    public static final MultiPolygonBuilder PROTOTYPE = new MultiPolygonBuilder();
 
-    private final ArrayList<PolygonBuilder> polygons = new ArrayList<>();
+    private final List<PolygonBuilder> polygons = new ArrayList<>();
 
-    private Orientation orientation = Orientation.RIGHT;
+    private final Orientation orientation;
 
+    /**
+     * Build a MultiPolygonBuilder with RIGHT orientation.
+     */
     public MultiPolygonBuilder() {
         this(Orientation.RIGHT);
     }
 
+    /**
+     * Build a MultiPolygonBuilder with an arbitrary orientation.
+     */
     public MultiPolygonBuilder(Orientation orientation) {
         this.orientation = orientation;
     }
 
+    /**
+     * Read from a stream.
+     */
+    public MultiPolygonBuilder(StreamInput in) throws IOException {
+        orientation = Orientation.readFrom(in);
+        int holes = in.readVInt();
+        for (int i = 0; i < holes; i++) {
+            polygon(new PolygonBuilder(in));
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        orientation.writeTo(out);
+        out.writeVInt(polygons.size());
+        for (PolygonBuilder polygon : polygons) {
+            polygon.writeTo(out);
+        }
+    }
+
     public Orientation orientation() {
         return this.orientation;
     }
@@ -70,7 +95,7 @@ public class MultiPolygonBuilder extends ShapeBuilder {
     /**
      * get the list of polygons
      */
-    public ArrayList<PolygonBuilder> polygons() {
+    public List<PolygonBuilder> polygons() {
         return polygons;
     }
 
@@ -134,23 +159,4 @@ public class MultiPolygonBuilder extends ShapeBuilder {
         return Objects.equals(polygons, other.polygons) &&
                 Objects.equals(orientation, other.orientation);
     }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        orientation.writeTo(out);
-        out.writeVInt(polygons.size());
-        for (PolygonBuilder polygon : polygons) {
-            polygon.writeTo(out);
-        }
-    }
-
-    @Override
-    public MultiPolygonBuilder readFrom(StreamInput in) throws IOException {
-        MultiPolygonBuilder polyBuilder = new MultiPolygonBuilder(Orientation.readFrom(in));
-        int holes = in.readVInt();
-        for (int i = 0; i < holes; i++) {
-            polyBuilder.polygon(PolygonBuilder.PROTOTYPE.readFrom(in));
-        }
-        return polyBuilder;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/PointBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/PointBuilder.java
index 30b7e370f22..fdd9826410a 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/PointBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/PointBuilder.java
@@ -30,9 +30,7 @@ import java.io.IOException;
 import java.util.Objects;
 
 public class PointBuilder extends ShapeBuilder {
-
     public static final GeoShapeType TYPE = GeoShapeType.POINT;
-    public static final PointBuilder PROTOTYPE = new PointBuilder();
 
     private Coordinate coordinate;
 
@@ -43,6 +41,18 @@ public class PointBuilder extends ShapeBuilder {
         this.coordinate = ZERO_ZERO;
     }
 
+    /**
+     * Read from a stream.
+     */
+    public PointBuilder(StreamInput in) throws IOException {
+        coordinate = readFromStream(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        writeCoordinateTo(coordinate, out);
+    }
+
     public PointBuilder coordinate(Coordinate coordinate) {
         this.coordinate = coordinate;
         return this;
@@ -91,14 +101,4 @@ public class PointBuilder extends ShapeBuilder {
         PointBuilder other = (PointBuilder) obj;
         return Objects.equals(coordinate, other.coordinate);
     }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        writeCoordinateTo(coordinate, out);
-    }
-
-    @Override
-    public PointBuilder readFrom(StreamInput in) throws IOException {
-        return new PointBuilder().coordinate(readCoordinateFrom(in));
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
index 4a9c8441072..9fad4fb8ef3 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
@@ -53,8 +53,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
 public class PolygonBuilder extends ShapeBuilder {
 
     public static final GeoShapeType TYPE = GeoShapeType.POLYGON;
-    public static final PolygonBuilder PROTOTYPE = new PolygonBuilder(new CoordinatesBuilder().coordinate(0.0, 0.0).coordinate(0.0, 1.0)
-            .coordinate(1.0, 0.0).coordinate(0.0, 0.0));
 
     private static final Coordinate[][] EMPTY = new Coordinate[0][];
 
@@ -64,7 +62,7 @@ public class PolygonBuilder extends ShapeBuilder {
     private LineStringBuilder shell;
 
     // List of line strings defining the holes of the polygon
-    private final ArrayList<LineStringBuilder> holes = new ArrayList<>();
+    private final List<LineStringBuilder> holes = new ArrayList<>();
 
     public PolygonBuilder(LineStringBuilder lineString, Orientation orientation, boolean coerce) {
         this.orientation = orientation;
@@ -87,6 +85,28 @@ public class PolygonBuilder extends ShapeBuilder {
         this(coordinates, Orientation.RIGHT);
     }
 
+    /**
+     * Read from a stream.
+     */
+    public PolygonBuilder(StreamInput in) throws IOException {
+        shell = new LineStringBuilder(in);
+        orientation = Orientation.readFrom(in);
+        int holes = in.readVInt();
+        for (int i = 0; i < holes; i++) {
+            hole(new LineStringBuilder(in));
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        shell.writeTo(out);
+        orientation.writeTo(out);
+        out.writeVInt(holes.size());
+        for (LineStringBuilder hole : holes) {
+            hole.writeTo(out);
+        }
+    }
+
     public Orientation orientation() {
         return this.orientation;
     }
@@ -314,7 +334,7 @@ public class PolygonBuilder extends ShapeBuilder {
 
         double shiftOffset = any.coordinate.x > DATELINE ? DATELINE : (any.coordinate.x < -DATELINE ? -DATELINE : 0);
         if (debugEnabled()) {
-            LOGGER.debug("shift: {[]}", shiftOffset);
+            LOGGER.debug("shift: [{}]", shiftOffset);
         }
 
         // run along the border of the component, collect the
@@ -383,18 +403,18 @@ public class PolygonBuilder extends ShapeBuilder {
         return coordinates;
     }
 
-    private static Coordinate[][][] buildCoordinates(ArrayList<ArrayList<Coordinate[]>> components) {
+    private static Coordinate[][][] buildCoordinates(List<List<Coordinate[]>> components) {
         Coordinate[][][] result = new Coordinate[components.size()][][];
         for (int i = 0; i < result.length; i++) {
-            ArrayList<Coordinate[]> component = components.get(i);
+            List<Coordinate[]> component = components.get(i);
             result[i] = component.toArray(new Coordinate[component.size()][]);
         }
 
         if(debugEnabled()) {
             for (int i = 0; i < result.length; i++) {
-                LOGGER.debug("Component {[]}:", i);
+                LOGGER.debug("Component [{}]:", i);
                 for (int j = 0; j < result[i].length; j++) {
-                    LOGGER.debug("\t" + Arrays.toString(result[i][j]));
+                    LOGGER.debug("\t{}", Arrays.toString(result[i][j]));
                 }
             }
         }
@@ -416,13 +436,13 @@ public class PolygonBuilder extends ShapeBuilder {
         return points;
     }
 
-    private static Edge[] edges(Edge[] edges, int numHoles, ArrayList<ArrayList<Coordinate[]>> components) {
+    private static Edge[] edges(Edge[] edges, int numHoles, List<List<Coordinate[]>> components) {
         ArrayList<Edge> mainEdges = new ArrayList<>(edges.length);
 
         for (int i = 0; i < edges.length; i++) {
             if (edges[i].component >= 0) {
                 int length = component(edges[i], -(components.size()+numHoles+1), mainEdges);
-                ArrayList<Coordinate[]> component = new ArrayList<>();
+                List<Coordinate[]> component = new ArrayList<>();
                 component.add(coordinates(edges[i], new Coordinate[length+1]));
                 components.add(component);
             }
@@ -432,19 +452,19 @@ public class PolygonBuilder extends ShapeBuilder {
     }
 
     private static Coordinate[][][] compose(Edge[] edges, Edge[] holes, int numHoles) {
-        final ArrayList<ArrayList<Coordinate[]>> components = new ArrayList<>();
+        final List<List<Coordinate[]>> components = new ArrayList<>();
         assign(holes, holes(holes, numHoles), numHoles, edges(edges, numHoles, components), components);
         return buildCoordinates(components);
     }
 
-    private static void assign(Edge[] holes, Coordinate[][] points, int numHoles, Edge[] edges, ArrayList<ArrayList<Coordinate[]>> components) {
+    private static void assign(Edge[] holes, Coordinate[][] points, int numHoles, Edge[] edges, List<List<Coordinate[]>> components) {
         // Assign Hole to related components
         // To find the new component the hole belongs to all intersections of the
         // polygon edges with a vertical line are calculated. This vertical line
         // is an arbitrary point of the hole. The polygon edge next to this point
         // is part of the polygon the hole belongs to.
         if (debugEnabled()) {
-            LOGGER.debug("Holes: " + Arrays.toString(holes));
+            LOGGER.debug("Holes: {}", Arrays.toString(holes));
         }
         for (int i = 0; i < numHoles; i++) {
             final Edge current = new Edge(holes[i].coordinate, holes[i].next);
@@ -464,9 +484,9 @@ public class PolygonBuilder extends ShapeBuilder {
             final int component = -edges[index].component - numHoles - 1;
 
             if(debugEnabled()) {
-                LOGGER.debug("\tposition ("+index+") of edge "+current+": " + edges[index]);
-                LOGGER.debug("\tComponent: " + component);
-                LOGGER.debug("\tHole intersections ("+current.coordinate.x+"): " + Arrays.toString(edges));
+                LOGGER.debug("\tposition ({}) of edge {}: {}", index, current, edges[index]);
+                LOGGER.debug("\tComponent: {}", component);
+                LOGGER.debug("\tHole intersections ({}): {}", current.coordinate.x, Arrays.toString(edges));
             }
 
             components.get(component).add(points[i]);
@@ -668,8 +688,8 @@ public class PolygonBuilder extends ShapeBuilder {
      *            number of points to use
      * @return the edges creates
      */
-    private static Edge[] concat(int component, boolean direction, Coordinate[] points, final int pointOffset, Edge[] edges, final int edgeOffset,
-            int length) {
+    private static Edge[] concat(int component, boolean direction, Coordinate[] points, final int pointOffset, Edge[] edges,
+            final int edgeOffset, int length) {
         assert edges.length >= length+edgeOffset;
         assert points.length >= length+pointOffset;
         edges[edgeOffset] = new Edge(points[pointOffset], null);
@@ -725,26 +745,4 @@ public class PolygonBuilder extends ShapeBuilder {
                 Objects.equals(holes, other.holes) &&
                 Objects.equals(orientation,  other.orientation);
     }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        shell.writeTo(out);
-        orientation.writeTo(out);
-        out.writeVInt(holes.size());
-        for (LineStringBuilder hole : holes) {
-            hole.writeTo(out);
-        }
-    }
-
-    @Override
-    public PolygonBuilder readFrom(StreamInput in) throws IOException {
-        LineStringBuilder shell = LineStringBuilder.PROTOTYPE.readFrom(in);
-        Orientation orientation = Orientation.readFrom(in);
-        PolygonBuilder polyBuilder = new PolygonBuilder(shell, orientation);
-        int holes = in.readVInt();
-        for (int i = 0; i < holes; i++) {
-            polyBuilder.hole(LineStringBuilder.PROTOTYPE.readFrom(in));
-        }
-        return polyBuilder;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java
index d0c73964575..a0d77d004da 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java
@@ -180,7 +180,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
         out.writeDouble(coordinate.y);
     }
 
-    protected Coordinate readCoordinateFrom(StreamInput in) throws IOException {
+    protected static Coordinate readFromStream(StreamInput in) throws IOException {
         return new Coordinate(in.readDouble(), in.readDouble());
     }
 
@@ -519,7 +519,8 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
             } else if (geometryCollections == null && GeoShapeType.GEOMETRYCOLLECTION == shapeType) {
                 throw new ElasticsearchParseException("geometries not included");
             } else if (radius != null && GeoShapeType.CIRCLE != shapeType) {
-                throw new ElasticsearchParseException("field [{}] is supported for [{}] only", CircleBuilder.FIELD_RADIUS, CircleBuilder.TYPE);
+                throw new ElasticsearchParseException("field [{}] is supported for [{}] only", CircleBuilder.FIELD_RADIUS,
+                        CircleBuilder.TYPE);
             }
 
             switch (shapeType) {
@@ -539,7 +540,8 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
 
         protected static void validatePointNode(CoordinateNode node) {
             if (node.isEmpty()) {
-                throw new ElasticsearchParseException("invalid number of points (0) provided when expecting a single coordinate ([lat, lng])");
+                throw new ElasticsearchParseException(
+                        "invalid number of points (0) provided when expecting a single coordinate ([lat, lng])");
             } else if (node.coordinate == null) {
                 if (node.children.isEmpty() == false) {
                     throw new ElasticsearchParseException("multipoint data provided when single point data expected.");
@@ -559,8 +561,9 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
         protected static EnvelopeBuilder parseEnvelope(CoordinateNode coordinates) {
             // validate the coordinate array for envelope type
             if (coordinates.children.size() != 2) {
-                throw new ElasticsearchParseException("invalid number of points [{}] provided for " +
-                        "geo_shape [{}] when expecting an array of 2 coordinates", coordinates.children.size(), GeoShapeType.ENVELOPE.shapename);
+                throw new ElasticsearchParseException(
+                        "invalid number of points [{}] provided for geo_shape [{}] when expecting an array of 2 coordinates",
+                        coordinates.children.size(), GeoShapeType.ENVELOPE.shapename);
             }
             // verify coordinate bounds, correct if necessary
             Coordinate uL = coordinates.children.get(0).coordinate;
@@ -604,7 +607,8 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
              * LineStringBuilder should throw a graceful exception if < 2 coordinates/points are provided
              */
             if (coordinates.children.size() < 2) {
-                throw new ElasticsearchParseException("invalid number of points in LineString (found [{}] - must be >= 2)", coordinates.children.size());
+                throw new ElasticsearchParseException("invalid number of points in LineString (found [{}] - must be >= 2)",
+                        coordinates.children.size());
             }
 
             CoordinatesBuilder line = new CoordinatesBuilder();
@@ -636,10 +640,10 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
                 throw new ElasticsearchParseException(error);
             }
 
-            int numValidPts;
-            if (coordinates.children.size() < (numValidPts = (coerce) ? 3 : 4)) {
-                throw new ElasticsearchParseException("invalid number of points in LinearRing (found [{}] - must be >= " +  numValidPts + ")(",
-                        coordinates.children.size());
+            int numValidPts = coerce ? 3 : 4;
+            if (coordinates.children.size() < numValidPts) {
+                throw new ElasticsearchParseException("invalid number of points in LinearRing (found [{}] - must be >= [{}])",
+                        coordinates.children.size(), numValidPts);
             }
 
             if (!coordinates.children.get(0).coordinate.equals(
@@ -655,7 +659,8 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
 
         protected static PolygonBuilder parsePolygon(CoordinateNode coordinates, final Orientation orientation, final boolean coerce) {
             if (coordinates.children == null || coordinates.children.isEmpty()) {
-                throw new ElasticsearchParseException("invalid LinearRing provided for type polygon. Linear ring must be an array of coordinates");
+                throw new ElasticsearchParseException(
+                        "invalid LinearRing provided for type polygon. Linear ring must be an array of coordinates");
             }
 
             LineStringBuilder shell = parseLinearRing(coordinates.children.get(0), coerce);
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilders.java b/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilders.java
index 1c828814431..5194510bcfb 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilders.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilders.java
@@ -21,6 +21,8 @@ package org.elasticsearch.common.geo.builders;
 
 import com.vividsolutions.jts.geom.Coordinate;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+
 import java.util.List;
 
 /**
@@ -137,4 +139,16 @@ public class ShapeBuilders {
     public static EnvelopeBuilder newEnvelope(Coordinate topLeft, Coordinate bottomRight) {
         return new EnvelopeBuilder(topLeft, bottomRight);
     }
+
+    public static void register(NamedWriteableRegistry namedWriteableRegistry) {
+        namedWriteableRegistry.register(ShapeBuilder.class, PointBuilder.TYPE.shapeName(), PointBuilder::new);
+        namedWriteableRegistry.register(ShapeBuilder.class, CircleBuilder.TYPE.shapeName(), CircleBuilder::new);
+        namedWriteableRegistry.register(ShapeBuilder.class, EnvelopeBuilder.TYPE.shapeName(), EnvelopeBuilder::new);
+        namedWriteableRegistry.register(ShapeBuilder.class, MultiPointBuilder.TYPE.shapeName(), MultiPointBuilder::new);
+        namedWriteableRegistry.register(ShapeBuilder.class, LineStringBuilder.TYPE.shapeName(), LineStringBuilder::new);
+        namedWriteableRegistry.register(ShapeBuilder.class, MultiLineStringBuilder.TYPE.shapeName(), MultiLineStringBuilder::new);
+        namedWriteableRegistry.register(ShapeBuilder.class, PolygonBuilder.TYPE.shapeName(), PolygonBuilder::new);
+        namedWriteableRegistry.register(ShapeBuilder.class, MultiPolygonBuilder.TYPE.shapeName(), MultiPolygonBuilder::new);
+        namedWriteableRegistry.register(ShapeBuilder.class, GeometryCollectionBuilder.TYPE.shapeName(), GeometryCollectionBuilder::new);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
index 5f3bd011dd9..b8132b4e870 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
@@ -59,6 +59,11 @@ public abstract class FilterStreamInput extends StreamInput {
         delegate.close();
     }
 
+    @Override
+    public int available() throws IOException {
+        return delegate.available();
+    }
+
     @Override
     public Version getVersion() {
         return delegate.getVersion();
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java
index e9aa52cf4d0..d786041af49 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java
@@ -74,6 +74,11 @@ public class InputStreamStreamInput extends StreamInput {
         is.close();
     }
 
+    @Override
+    public int available() throws IOException {
+        return is.available();
+    }
+
     @Override
     public int read() throws IOException {
         return is.read();
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java
index a6d17089652..c683573df7a 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java
@@ -36,7 +36,12 @@ public class NamedWriteableAwareStreamInput extends FilterStreamInput {
     @Override
     <C> C readNamedWriteable(Class<C> categoryClass) throws IOException {
         String name = readString();
-        NamedWriteable<? extends C> namedWriteable = namedWriteableRegistry.getPrototype(categoryClass, name);
-        return namedWriteable.readFrom(this);
+        Writeable.Reader<? extends C> reader = namedWriteableRegistry.getReader(categoryClass, name);
+        C c = reader.read(this);
+        if (c == null) {
+            throw new IOException(
+                    "Writeable.Reader [" + reader + "] returned null which is not allowed and probably means it screwed up the stream.");
+        }
+        return c;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableRegistry.java b/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableRegistry.java
index 42014786749..5a3de923bde 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableRegistry.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableRegistry.java
@@ -31,54 +31,70 @@ public class NamedWriteableRegistry {
     private final Map<Class<?>, InnerRegistry<?>> registry = new HashMap<>();
 
     /**
-     * Registers a {@link NamedWriteable} prototype given its category
+     * Register a {@link NamedWriteable} given its category, its name, and a function to read it from the stream.
+     *
+     * This method suppresses the rawtypes warning because it intentionally using NamedWriteable instead of {@code NamedWriteable<T>} so it
+     * is easier to use and because we might be able to drop the type parameter from NamedWriteable entirely some day.
      */
-    public synchronized <T> void registerPrototype(Class<T> categoryClass, NamedWriteable<? extends T> namedWriteable) {
+    @SuppressWarnings("rawtypes")
+    public synchronized <T extends NamedWriteable> void register(Class<T> categoryClass, String name,
+            Writeable.Reader<? extends T> reader) {
         @SuppressWarnings("unchecked")
-        InnerRegistry<T> innerRegistry = (InnerRegistry<T>)registry.get(categoryClass);
+        InnerRegistry<T> innerRegistry = (InnerRegistry<T>) registry.get(categoryClass);
         if (innerRegistry == null) {
             innerRegistry = new InnerRegistry<>(categoryClass);
             registry.put(categoryClass, innerRegistry);
         }
-        innerRegistry.registerPrototype(namedWriteable);
+        innerRegistry.register(name, reader);
+    }
+
+    /**
+     * Registers a {@link NamedWriteable} prototype given its category.
+     * @deprecated Prefer {@link #register(Class, String, org.elasticsearch.common.io.stream.Writeable.Reader)}
+     */
+    @Deprecated
+    @SuppressWarnings("rawtypes") // TODO remove this method entirely before 5.0.0 GA
+    public synchronized <T extends NamedWriteable> void registerPrototype(Class<T> categoryClass,
+            NamedWriteable<? extends T> namedWriteable) {
+        register(categoryClass, namedWriteable.getWriteableName(), namedWriteable::readFrom);
     }
 
     /**
      * Returns a prototype of the {@link NamedWriteable} object identified by the name provided as argument and its category
      */
-    public synchronized <T> NamedWriteable<? extends T> getPrototype(Class<T> categoryClass, String name) {
+    public synchronized <T> Writeable.Reader<? extends T> getReader(Class<T> categoryClass, String name) {
         @SuppressWarnings("unchecked")
         InnerRegistry<T> innerRegistry = (InnerRegistry<T>)registry.get(categoryClass);
         if (innerRegistry == null) {
             throw new IllegalArgumentException("unknown named writeable category [" + categoryClass.getName() + "]");
         }
-        return innerRegistry.getPrototype(name);
+        return innerRegistry.getReader(name);
     }
 
     private static class InnerRegistry<T> {
 
-        private final Map<String, NamedWriteable<? extends T>> registry = new HashMap<>();
+        private final Map<String, Writeable.Reader<? extends T>> registry = new HashMap<>();
         private final Class<T> categoryClass;
 
         private InnerRegistry(Class<T> categoryClass) {
             this.categoryClass = categoryClass;
         }
 
-        private void registerPrototype(NamedWriteable<? extends T> namedWriteable) {
-            NamedWriteable<? extends T> existingNamedWriteable = registry.get(namedWriteable.getWriteableName());
-            if (existingNamedWriteable != null) {
-                throw new IllegalArgumentException("named writeable of type [" + namedWriteable.getClass().getName() + "] with name [" + namedWriteable.getWriteableName() + "] " +
-                        "is already registered by type [" + existingNamedWriteable.getClass().getName() + "] within category [" + categoryClass.getName() + "]");
+        private void register(String name, Writeable.Reader<? extends T> reader) {
+            Writeable.Reader<? extends T> existingReader = registry.get(name);
+            if (existingReader != null) {
+                throw new IllegalArgumentException(
+                        "named writeable [" + categoryClass.getName() + "][" + name + "] is already registered by [" + reader + "]");
             }
-            registry.put(namedWriteable.getWriteableName(), namedWriteable);
+            registry.put(name, reader);
         }
 
-        private NamedWriteable<? extends T> getPrototype(String name) {
-            NamedWriteable<? extends T> namedWriteable = registry.get(name);
-            if (namedWriteable == null) {
-                throw new IllegalArgumentException("unknown named writeable with name [" + name + "] within category [" + categoryClass.getName() + "]");
+        private Writeable.Reader<? extends T> getReader(String name) {
+            Writeable.Reader<? extends T> reader = registry.get(name);
+            if (reader == null) {
+                throw new IllegalArgumentException("unknown named writeable [" + categoryClass.getName() + "][" + name + "]");
             }
-            return namedWriteable;
+            return reader;
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index 8eda42ae9be..a5750fcc542 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -37,10 +37,13 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
-import org.elasticsearch.search.rescore.RescoreBuilder;
-import org.elasticsearch.tasks.Task;
 import org.elasticsearch.search.aggregations.AggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
+import org.elasticsearch.search.rescore.RescoreBuilder;
+import org.elasticsearch.search.sort.SortBuilder;
+import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.suggest.phrase.SmoothingModel;
+import org.elasticsearch.tasks.Task;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
@@ -282,6 +285,14 @@ public abstract class StreamInput extends InputStream {
         return null;
     }
 
+    @Nullable
+    public Float readOptionalFloat() throws IOException {
+        if (readBoolean()) {
+            return readFloat();
+        }
+        return null;
+    }
+
     @Nullable
     public Integer readOptionalVInt() throws IOException {
         if (readBoolean()) {
@@ -362,6 +373,9 @@ public abstract class StreamInput extends InputStream {
     @Override
     public abstract void close() throws IOException;
 
+    @Override
+    public abstract int available() throws IOException;
+
     public String[] readStringArray() throws IOException {
         int size = readVInt();
         if (size == 0) {
@@ -552,6 +566,19 @@ public abstract class StreamInput extends InputStream {
         }
     }
 
+    public <T extends Writeable> T readOptionalWriteable(Writeable.Reader<T> reader) throws IOException {
+        if (readBoolean()) {
+            T t = reader.read(this);
+            if (t == null) {
+                throw new IOException("Writeable.Reader [" + reader
+                        + "] returned null which is not allowed and probably means it screwed up the stream.");
+            }
+            return t;
+        } else {
+            return null;
+        }
+    }
+
     public <T extends Throwable> T readThrowable() throws IOException {
         if (readBoolean()) {
             int key = readVInt();
@@ -666,24 +693,34 @@ public abstract class StreamInput extends InputStream {
     /**
      * Reads a {@link AggregatorBuilder} from the current stream
      */
-    public AggregatorBuilder readAggregatorFactory() throws IOException {
+    public AggregatorBuilder<?> readAggregatorFactory() throws IOException {
         return readNamedWriteable(AggregatorBuilder.class);
     }
 
     /**
      * Reads a {@link PipelineAggregatorBuilder} from the current stream
      */
-    public PipelineAggregatorBuilder readPipelineAggregatorFactory() throws IOException {
+    public PipelineAggregatorBuilder<?> readPipelineAggregatorFactory() throws IOException {
         return readNamedWriteable(PipelineAggregatorBuilder.class);
     }
 
     /**
      * Reads a {@link QueryBuilder} from the current stream
      */
-    public QueryBuilder readQuery() throws IOException {
+    public QueryBuilder<?> readQuery() throws IOException {
         return readNamedWriteable(QueryBuilder.class);
     }
 
+    /**
+     * Reads an optional {@link QueryBuilder}.
+     */
+    public QueryBuilder<?> readOptionalQuery() throws IOException {
+        if (readBoolean()) {
+            return readNamedWriteable(QueryBuilder.class);
+        }
+        return null;
+    }
+
     /**
      * Reads a {@link ShapeBuilder} from the current stream
      */
@@ -698,6 +735,20 @@ public abstract class StreamInput extends InputStream {
         return readNamedWriteable(RescoreBuilder.class);
     }
 
+    /**
+     * Reads a {@link SuggestionBuilder} from the current stream
+     */
+    public SuggestionBuilder<?> readSuggestion() throws IOException {
+        return readNamedWriteable(SuggestionBuilder.class);
+    }
+
+    /**
+     * Reads a {@link SortBuilder} from the current stream
+     */
+    public SortBuilder<?> readSortBuilder() throws IOException {
+        return readNamedWriteable(SortBuilder.class);
+    }
+
     /**
      * Reads a {@link org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder} from the current stream
      */
@@ -705,6 +756,13 @@ public abstract class StreamInput extends InputStream {
         return readNamedWriteable(ScoreFunctionBuilder.class);
     }
 
+    /**
+     * Reads a {@link SmoothingModel} from the current stream
+     */
+    public SmoothingModel readPhraseSuggestionSmoothingModel() throws IOException {
+        return readNamedWriteable(SmoothingModel.class);
+    }
+
     /**
      * Reads a {@link Task.Status} from the current stream.
      */
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
index 864da006bf0..6b7607a3e70 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
@@ -36,10 +36,13 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
-import org.elasticsearch.search.rescore.RescoreBuilder;
-import org.elasticsearch.tasks.Task;
 import org.elasticsearch.search.aggregations.AggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
+import org.elasticsearch.search.rescore.RescoreBuilder;
+import org.elasticsearch.search.sort.SortBuilder;
+import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.suggest.phrase.SmoothingModel;
+import org.elasticsearch.tasks.Task;
 import org.joda.time.ReadableInstant;
 
 import java.io.EOFException;
@@ -238,6 +241,15 @@ public abstract class StreamOutput extends OutputStream {
         }
     }
 
+    public void writeOptionalFloat(@Nullable Float floatValue) throws IOException {
+        if (floatValue == null) {
+            writeBoolean(false);
+        } else {
+            writeBoolean(true);
+            writeFloat(floatValue);
+        }
+    }
+
     public void writeOptionalText(@Nullable Text text) throws IOException {
         if (text == null) {
             writeInt(-1);
@@ -520,6 +532,15 @@ public abstract class StreamOutput extends OutputStream {
         }
     }
 
+    public void writeOptionalWriteable(@Nullable Writeable<?> writeable) throws IOException {
+        if (writeable != null) {
+            writeBoolean(true);
+            writeable.writeTo(this);
+        } else {
+            writeBoolean(false);
+        }
+    }
+
     public void writeThrowable(Throwable throwable) throws IOException {
         if (throwable == null) {
             writeBoolean(false);
@@ -642,7 +663,7 @@ public abstract class StreamOutput extends OutputStream {
     /**
      * Writes a {@link NamedWriteable} to the current stream, by first writing its name and then the object itself
      */
-    void writeNamedWriteable(NamedWriteable namedWriteable) throws IOException {
+    void writeNamedWriteable(NamedWriteable<?> namedWriteable) throws IOException {
         writeString(namedWriteable.getWriteableName());
         namedWriteable.writeTo(this);
     }
@@ -664,10 +685,22 @@ public abstract class StreamOutput extends OutputStream {
     /**
      * Writes a {@link QueryBuilder} to the current stream
      */
-    public void writeQuery(QueryBuilder queryBuilder) throws IOException {
+    public void writeQuery(QueryBuilder<?> queryBuilder) throws IOException {
         writeNamedWriteable(queryBuilder);
     }
 
+    /**
+     * Write an optional {@link QueryBuilder} to the stream.
+     */
+    public void writeOptionalQuery(@Nullable QueryBuilder<?> queryBuilder) throws IOException {
+        if (queryBuilder == null) {
+            writeBoolean(false);
+        } else {
+            writeBoolean(true);
+            writeQuery(queryBuilder);
+        }
+    }
+
     /**
      * Writes a {@link ShapeBuilder} to the current stream
      */
@@ -682,6 +715,13 @@ public abstract class StreamOutput extends OutputStream {
         writeNamedWriteable(scoreFunctionBuilder);
     }
 
+    /**
+     * Writes the given {@link SmoothingModel} to the stream
+     */
+    public void writePhraseSuggestionSmoothingModel(SmoothingModel smoothinModel) throws IOException {
+        writeNamedWriteable(smoothinModel);
+    }
+
     /**
      * Writes a {@link Task.Status} to the current stream.
      */
@@ -713,4 +753,19 @@ public abstract class StreamOutput extends OutputStream {
     public void writeRescorer(RescoreBuilder<?> rescorer) throws IOException {
         writeNamedWriteable(rescorer);
     }
+
+    /**
+     * Writes a {@link SuggestionBuilder} to the current stream
+     */
+    public void writeSuggestion(SuggestionBuilder<?> suggestion) throws IOException {
+        writeNamedWriteable(suggestion);
+    }
+
+    /**
+     * Writes a {@link SortBuilder} to the current stream
+     */
+    public void writeSortBuilder(SortBuilder<?> sort) throws IOException {
+        writeNamedWriteable(sort);
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamableReader.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamableReader.java
index 6bb1c5653f3..bd37f5ed47b 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamableReader.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamableReader.java
@@ -23,10 +23,7 @@ import java.io.IOException;
 /**
  * Implementers can be read from {@linkplain StreamInput} by calling their {@link #readFrom(StreamInput)} method.
  *
- * It is common for implementers of this interface to declare a <code>public static final</code> instance of themselves named PROTOTYPE so
- * users can call {@linkplain #readFrom(StreamInput)} on it. It is also fairly typical for readFrom to be implemented as a method that just
- * calls a constructor that takes {@linkplain StreamInput} as a parameter. This allows the fields in the implementer to be
- * <code>final</code>.
+ * Implementers of this interface that also implement {@link Writeable} should see advice there on how to do so.
  */
 public interface StreamableReader<T> {
     /**
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java b/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java
index 9ff3de736c5..75c1f28c39c 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java
@@ -31,10 +31,31 @@ import java.io.IOException;
  *
  * Prefer implementing this interface over implementing {@link Streamable} where possible. Lots of code depends on {@linkplain Streamable}
  * so this isn't always possible.
+ *
+ * The fact that this interface extends {@link StreamableReader} should be consider vestigial. Instead of using its
+ * {@link #readFrom(StreamInput)} method you should prefer using the Reader interface as a reference to a constructor that takes
+ * {@link StreamInput}. The reasoning behind this is that most "good" readFrom implementations just delegated to such a constructor anyway
+ * and they required an unsightly PROTOTYPE object.
  */
-public interface Writeable<T> extends StreamableReader<T> {
+public interface Writeable<T> extends StreamableReader<T> { // TODO remove extends StreamableReader<T> from this interface, and remove <T>
     /**
      * Write this into the {@linkplain StreamOutput}.
      */
     void writeTo(StreamOutput out) throws IOException;
+
+    @Override
+    default T readFrom(StreamInput in) throws IOException {
+        // See class javadoc for reasoning
+        throw new UnsupportedOperationException("Prefer calling a constructor that takes a StreamInput to calling readFrom.");
+    }
+
+    /**
+     * Reference to a method that can read some object from a stream. By convention this is a constructor that takes
+     * {@linkplain StreamInput} as an argument for most classes and a static method for things like enums. Returning null from one of these
+     * is always wrong - for that we use methods like {@link StreamInput#readOptionalWriteable(Reader)}.
+     */
+    @FunctionalInterface
+    interface Reader<R> {
+        R read(StreamInput t) throws IOException;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java b/core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
index 870b5f61466..b792a85d34c 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.common.logging;
 
+import org.elasticsearch.common.SuppressLoggerChecks;
+
 /**
  * A logger that logs deprecation notices.
  */
@@ -45,6 +47,7 @@ public class DeprecationLogger {
     /**
      * Logs a deprecated message.
      */
+    @SuppressLoggerChecks(reason = "safely delegates to logger")
     public void deprecated(String msg, Object... params) {
         logger.debug(msg, params);
     }
diff --git a/core/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java b/core/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java
index 94ade9334d7..c0951c47df1 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java
@@ -21,6 +21,7 @@ package org.elasticsearch.common.logging;
 
 import org.apache.log4j.Logger;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 
 import java.util.Locale;
 
@@ -30,9 +31,10 @@ import java.util.Locale;
 public abstract class ESLoggerFactory {
 
     public static final Setting<LogLevel> LOG_DEFAULT_LEVEL_SETTING =
-            new Setting<>("logger.level", LogLevel.INFO.name(), LogLevel::parse, false, Setting.Scope.CLUSTER);
+        new Setting<>("logger.level", LogLevel.INFO.name(), LogLevel::parse, Property.NodeScope);
     public static final Setting<LogLevel> LOG_LEVEL_SETTING =
-            Setting.prefixKeySetting("logger.", LogLevel.INFO.name(), LogLevel::parse, true, Setting.Scope.CLUSTER);
+        Setting.prefixKeySetting("logger.", LogLevel.INFO.name(), LogLevel::parse,
+            Property.Dynamic, Property.NodeScope);
 
     public static ESLogger getLogger(String prefix, String name) {
         prefix = prefix == null ? null : prefix.intern();
diff --git a/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java b/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
index 28feca13c02..da628b09d2b 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
@@ -110,9 +110,7 @@ public class LogConfigurator {
         if (resolveConfig) {
             resolveConfig(environment, settingsBuilder);
         }
-        settingsBuilder
-                .putProperties("elasticsearch.", BootstrapInfo.getSystemProperties())
-                .putProperties("es.", BootstrapInfo.getSystemProperties());
+        settingsBuilder.putProperties("es.", BootstrapInfo.getSystemProperties());
         // add custom settings after config was added so that they are not overwritten by config
         settingsBuilder.put(settings);
         settingsBuilder.replacePropertyPlaceholders();
diff --git a/core/src/main/java/org/elasticsearch/common/logging/TerminalAppender.java b/core/src/main/java/org/elasticsearch/common/logging/TerminalAppender.java
index 7031a62a999..e967ad9d79e 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/TerminalAppender.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/TerminalAppender.java
@@ -22,7 +22,7 @@ package org.elasticsearch.common.logging;
 
 import org.apache.log4j.AppenderSkeleton;
 import org.apache.log4j.spi.LoggingEvent;
-import org.elasticsearch.common.cli.Terminal;
+import org.elasticsearch.cli.Terminal;
 
 /**
  * TerminalAppender logs event to Terminal.DEFAULT. It is used for example by the PluginCli.
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
index 39f34ad867e..8508a8a2e40 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
@@ -111,7 +111,7 @@ public class Lucene {
         try {
             return Version.parse(version);
         } catch (ParseException e) {
-            logger.warn("no version match {}, default to {}", version, defaultVersion, e);
+            logger.warn("no version match {}, default to {}", e, version, defaultVersion);
             return defaultVersion;
         }
     }
@@ -235,11 +235,7 @@ public class Lucene {
             @Override
             protected Object doBody(String segmentFileName) throws IOException {
                 try (IndexInput input = directory.openInput(segmentFileName, IOContext.READ)) {
-                    final int format = input.readInt();
-                    if (format == CodecUtil.CODEC_MAGIC) {
-                        CodecUtil.checksumEntireFile(input);
-                    }
-                    // legacy....
+                    CodecUtil.checksumEntireFile(input);
                 }
                 return null;
             }
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java
index 754d76fed27..52de9a7e5db 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java
@@ -134,7 +134,7 @@ public class MultiPhrasePrefixQuery extends Query {
         if (termArrays.isEmpty()) {
             return new MatchNoDocsQuery();
         }
-        MultiPhraseQuery query = new MultiPhraseQuery();
+        MultiPhraseQuery.Builder query = new MultiPhraseQuery.Builder();
         query.setSlop(slop);
         int sizeMinus1 = termArrays.size() - 1;
         for (int i = 0; i < sizeMinus1; i++) {
@@ -153,7 +153,7 @@ public class MultiPhrasePrefixQuery extends Query {
             return Queries.newMatchNoDocsQuery();
         }
         query.add(terms.toArray(Term.class), position);
-        return query.rewrite(reader);
+        return query.build();
     }
 
     private void getPrefixTerms(ObjectHashSet<Term> terms, final Term prefix, final IndexReader reader) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/common/network/IfConfig.java b/core/src/main/java/org/elasticsearch/common/network/IfConfig.java
index ea33274fad1..282f348c81b 100644
--- a/core/src/main/java/org/elasticsearch/common/network/IfConfig.java
+++ b/core/src/main/java/org/elasticsearch/common/network/IfConfig.java
@@ -31,14 +31,14 @@ import java.net.SocketException;
 import java.util.List;
 import java.util.Locale;
 
-/** 
+/**
  * Simple class to log {@code ifconfig}-style output at DEBUG logging.
  */
 final class IfConfig {
 
-    private static final ESLogger logger = Loggers.getLogger(IfConfig.class);    
+    private static final ESLogger logger = Loggers.getLogger(IfConfig.class);
     private static final String INDENT = "        ";
-    
+
     /** log interface configuration at debug level, if its enabled */
     static void logIfNecessary() {
         if (logger.isDebugEnabled()) {
@@ -49,7 +49,7 @@ final class IfConfig {
             }
         }
     }
-    
+
     /** perform actual logging: might throw exception if things go wrong */
     private static void doLogging() throws IOException {
         StringBuilder msg = new StringBuilder();
@@ -59,14 +59,14 @@ final class IfConfig {
             // ordinary name
             msg.append(nic.getName());
             msg.append(System.lineSeparator());
-            
+
             // display name (e.g. on windows)
             if (!nic.getName().equals(nic.getDisplayName())) {
                 msg.append(INDENT);
                 msg.append(nic.getDisplayName());
                 msg.append(System.lineSeparator());
             }
-            
+
             // addresses: v4 first, then v6
             List<InterfaceAddress> addresses = nic.getInterfaceAddresses();
             for (InterfaceAddress address : addresses) {
@@ -76,7 +76,7 @@ final class IfConfig {
                     msg.append(System.lineSeparator());
                 }
             }
-            
+
             for (InterfaceAddress address : addresses) {
                 if (address.getAddress() instanceof Inet6Address) {
                     msg.append(INDENT);
@@ -84,7 +84,7 @@ final class IfConfig {
                     msg.append(System.lineSeparator());
                 }
             }
-            
+
             // hardware address
             byte hardware[] = nic.getHardwareAddress();
             if (hardware != null) {
@@ -98,19 +98,19 @@ final class IfConfig {
                 }
                 msg.append(System.lineSeparator());
             }
-             
+
             // attributes
             msg.append(INDENT);
             msg.append(formatFlags(nic));
             msg.append(System.lineSeparator());
         }
-        logger.debug("configuration:" + System.lineSeparator() + "{}", msg.toString());
+        logger.debug("configuration:{}{}", System.lineSeparator(), msg);
     }
-    
+
     /** format internet address: java's default doesn't include everything useful */
     private static String formatAddress(InterfaceAddress interfaceAddress) throws IOException {
         StringBuilder sb = new StringBuilder();
-        
+
         InetAddress address = interfaceAddress.getAddress();
         if (address instanceof Inet6Address) {
             sb.append("inet6 ");
@@ -122,10 +122,10 @@ final class IfConfig {
             sb.append(NetworkAddress.formatAddress(address));
             int netmask = 0xFFFFFFFF << (32 - interfaceAddress.getNetworkPrefixLength());
             sb.append(" netmask:" + NetworkAddress.formatAddress(InetAddress.getByAddress(new byte[] {
-                    (byte)(netmask >>> 24), 
-                    (byte)(netmask >>> 16 & 0xFF), 
-                    (byte)(netmask >>> 8 & 0xFF), 
-                    (byte)(netmask & 0xFF) 
+                    (byte)(netmask >>> 24),
+                    (byte)(netmask >>> 16 & 0xFF),
+                    (byte)(netmask >>> 8 & 0xFF),
+                    (byte)(netmask & 0xFF)
             })));
             InetAddress broadcast = interfaceAddress.getBroadcast();
             if (broadcast != null) {
@@ -141,7 +141,7 @@ final class IfConfig {
         }
         return sb.toString();
     }
-    
+
     /** format network interface flags */
     private static String formatFlags(NetworkInterface nic) throws SocketException {
         StringBuilder flags = new StringBuilder();
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
index b0598469d3a..712bdbe99ab 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
@@ -22,14 +22,16 @@ package org.elasticsearch.common.network;
 import java.util.Arrays;
 import java.util.List;
 
+import org.elasticsearch.action.support.replication.ReplicationTask;
 import org.elasticsearch.client.transport.TransportClientNodesService;
 import org.elasticsearch.client.transport.support.TransportProxyClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.Setting.Scope;
 import org.elasticsearch.common.util.ExtensionPoint;
 import org.elasticsearch.http.HttpServer;
 import org.elasticsearch.http.HttpServerTransport;
@@ -139,6 +141,7 @@ import org.elasticsearch.rest.action.template.RestPutSearchTemplateAction;
 import org.elasticsearch.rest.action.termvectors.RestMultiTermVectorsAction;
 import org.elasticsearch.rest.action.termvectors.RestTermVectorsAction;
 import org.elasticsearch.rest.action.update.RestUpdateAction;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.local.LocalTransport;
@@ -155,10 +158,11 @@ public class NetworkModule extends AbstractModule {
     public static final String LOCAL_TRANSPORT = "local";
     public static final String NETTY_TRANSPORT = "netty";
 
-    public static final Setting<String> HTTP_TYPE_SETTING = Setting.simpleString("http.type", false, Scope.CLUSTER);
-    public static final Setting<Boolean> HTTP_ENABLED = Setting.boolSetting("http.enabled", true, false, Scope.CLUSTER);
-    public static final Setting<String> TRANSPORT_SERVICE_TYPE_SETTING = Setting.simpleString("transport.service.type", false, Scope.CLUSTER);
-    public static final Setting<String> TRANSPORT_TYPE_SETTING = Setting.simpleString("transport.type", false, Scope.CLUSTER);
+    public static final Setting<String> HTTP_TYPE_SETTING = Setting.simpleString("http.type", Property.NodeScope);
+    public static final Setting<Boolean> HTTP_ENABLED = Setting.boolSetting("http.enabled", true, Property.NodeScope);
+    public static final Setting<String> TRANSPORT_SERVICE_TYPE_SETTING =
+        Setting.simpleString("transport.service.type", Property.NodeScope);
+    public static final Setting<String> TRANSPORT_TYPE_SETTING = Setting.simpleString("transport.type", Property.NodeScope);
 
 
 
@@ -325,6 +329,7 @@ public class NetworkModule extends AbstractModule {
         registerTransportService(NETTY_TRANSPORT, TransportService.class);
         registerTransport(LOCAL_TRANSPORT, LocalTransport.class);
         registerTransport(NETTY_TRANSPORT, NettyTransport.class);
+        registerTaskStatus(ReplicationTask.Status.NAME, ReplicationTask.Status::new);
 
         if (transportClient == false) {
             registerHttpTransport(NETTY_TRANSPORT, NettyHttpServerTransport.class);
@@ -370,6 +375,10 @@ public class NetworkModule extends AbstractModule {
         }
     }
 
+    public void registerTaskStatus(String name, Writeable.Reader<? extends Task.Status> reader) {
+        namedWriteableRegistry.register(Task.Status.class, name, reader);
+    }
+
     @Override
     protected void configure() {
         bind(NetworkService.class).toInstance(networkService);
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkService.java b/core/src/main/java/org/elasticsearch/common/network/NetworkService.java
index 5e8dbc4dcad..ff1f3912cc5 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkService.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkService.java
@@ -22,6 +22,7 @@ package org.elasticsearch.common.network;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
@@ -34,6 +35,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 
 /**
  *
@@ -43,24 +45,33 @@ public class NetworkService extends AbstractComponent {
     /** By default, we bind to loopback interfaces */
     public static final String DEFAULT_NETWORK_HOST = "_local_";
 
-    public static final Setting<List<String>> GLOBAL_NETWORK_HOST_SETTING = Setting.listSetting("network.host", Arrays.asList(DEFAULT_NETWORK_HOST),
-            s -> s, false, Setting.Scope.CLUSTER);
-    public static final Setting<List<String>> GLOBAL_NETWORK_BINDHOST_SETTING = Setting.listSetting("network.bind_host", GLOBAL_NETWORK_HOST_SETTING,
-            s -> s, false, Setting.Scope.CLUSTER);
-    public static final Setting<List<String>> GLOBAL_NETWORK_PUBLISHHOST_SETTING = Setting.listSetting("network.publish_host", GLOBAL_NETWORK_HOST_SETTING,
-            s -> s, false, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> NETWORK_SERVER = Setting.boolSetting("network.server", true, false, Setting.Scope.CLUSTER);
+    public static final Setting<List<String>> GLOBAL_NETWORK_HOST_SETTING =
+        Setting.listSetting("network.host", Arrays.asList(DEFAULT_NETWORK_HOST), Function.identity(), Property.NodeScope);
+    public static final Setting<List<String>> GLOBAL_NETWORK_BINDHOST_SETTING =
+        Setting.listSetting("network.bind_host", GLOBAL_NETWORK_HOST_SETTING, Function.identity(), Property.NodeScope);
+    public static final Setting<List<String>> GLOBAL_NETWORK_PUBLISHHOST_SETTING =
+        Setting.listSetting("network.publish_host", GLOBAL_NETWORK_HOST_SETTING, Function.identity(), Property.NodeScope);
+    public static final Setting<Boolean> NETWORK_SERVER = Setting.boolSetting("network.server", true, Property.NodeScope);
 
     public static final class TcpSettings {
-        public static final Setting<Boolean> TCP_NO_DELAY = Setting.boolSetting("network.tcp.no_delay", true, false, Setting.Scope.CLUSTER);
-        public static final Setting<Boolean> TCP_KEEP_ALIVE = Setting.boolSetting("network.tcp.keep_alive", true, false, Setting.Scope.CLUSTER);
-        public static final Setting<Boolean> TCP_REUSE_ADDRESS = Setting.boolSetting("network.tcp.reuse_address", NetworkUtils.defaultReuseAddress(), false, Setting.Scope.CLUSTER);
-        public static final Setting<ByteSizeValue> TCP_SEND_BUFFER_SIZE = Setting.byteSizeSetting("network.tcp.send_buffer_size", new ByteSizeValue(-1), false, Setting.Scope.CLUSTER);
-        public static final Setting<ByteSizeValue> TCP_RECEIVE_BUFFER_SIZE = Setting.byteSizeSetting("network.tcp.receive_buffer_size", new ByteSizeValue(-1), false, Setting.Scope.CLUSTER);
-        public static final Setting<Boolean> TCP_BLOCKING = Setting.boolSetting("network.tcp.blocking", false, false, Setting.Scope.CLUSTER);
-        public static final Setting<Boolean> TCP_BLOCKING_SERVER = Setting.boolSetting("network.tcp.blocking_server", TCP_BLOCKING, false, Setting.Scope.CLUSTER);
-        public static final Setting<Boolean> TCP_BLOCKING_CLIENT = Setting.boolSetting("network.tcp.blocking_client", TCP_BLOCKING, false, Setting.Scope.CLUSTER);
-        public static final Setting<TimeValue> TCP_CONNECT_TIMEOUT = Setting.timeSetting("network.tcp.connect_timeout", new TimeValue(30, TimeUnit.SECONDS), false, Setting.Scope.CLUSTER);
+        public static final Setting<Boolean> TCP_NO_DELAY =
+            Setting.boolSetting("network.tcp.no_delay", true, Property.NodeScope);
+        public static final Setting<Boolean> TCP_KEEP_ALIVE =
+            Setting.boolSetting("network.tcp.keep_alive", true, Property.NodeScope);
+        public static final Setting<Boolean> TCP_REUSE_ADDRESS =
+            Setting.boolSetting("network.tcp.reuse_address", NetworkUtils.defaultReuseAddress(), Property.NodeScope);
+        public static final Setting<ByteSizeValue> TCP_SEND_BUFFER_SIZE =
+            Setting.byteSizeSetting("network.tcp.send_buffer_size", new ByteSizeValue(-1), Property.NodeScope);
+        public static final Setting<ByteSizeValue> TCP_RECEIVE_BUFFER_SIZE =
+            Setting.byteSizeSetting("network.tcp.receive_buffer_size", new ByteSizeValue(-1), Property.NodeScope);
+        public static final Setting<Boolean> TCP_BLOCKING =
+            Setting.boolSetting("network.tcp.blocking", false, Property.NodeScope);
+        public static final Setting<Boolean> TCP_BLOCKING_SERVER =
+            Setting.boolSetting("network.tcp.blocking_server", TCP_BLOCKING, Property.NodeScope);
+        public static final Setting<Boolean> TCP_BLOCKING_CLIENT =
+            Setting.boolSetting("network.tcp.blocking_client", TCP_BLOCKING, Property.NodeScope);
+        public static final Setting<TimeValue> TCP_CONNECT_TIMEOUT =
+            Setting.timeSetting("network.tcp.connect_timeout", new TimeValue(30, TimeUnit.SECONDS), Property.NodeScope);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java b/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java
index a210f3ae6d5..70e6807cb92 100644
--- a/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java
+++ b/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java
@@ -76,8 +76,9 @@ public class PropertyPlaceholder {
      * @param placeholderResolver the <code>PlaceholderResolver</code> to use for replacement.
      * @return the supplied value with placeholders replaced inline.
      */
-    public String replacePlaceholders(String value, PlaceholderResolver placeholderResolver) {
-        Objects.requireNonNull(value, "Argument 'value' must not be null.");
+    public String replacePlaceholders(String key, String value, PlaceholderResolver placeholderResolver) {
+        Objects.requireNonNull(key);
+        Objects.requireNonNull(value, "value can not be null for [" + key + "]");
         return parseStringValue(value, placeholderResolver, new HashSet<String>());
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
index 453fc3f9a36..358706c9d3f 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
@@ -19,7 +19,11 @@
 
 package org.elasticsearch.common.settings;
 
+import org.apache.lucene.search.spell.LevensteinDistance;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.regex.Regex;
 
@@ -30,37 +34,41 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 /**
  * A basic setting service that can be used for per-index and per-cluster settings.
  * This service offers transactional application of updates settings.
  */
 public abstract class AbstractScopedSettings extends AbstractComponent {
+    public static final String ARCHIVED_SETTINGS_PREFIX = "archived.";
     private Settings lastSettingsApplied = Settings.EMPTY;
     private final List<SettingUpdater<?>> settingUpdaters = new CopyOnWriteArrayList<>();
     private final Map<String, Setting<?>> complexMatchers;
     private final Map<String, Setting<?>> keySettings;
-    private final Setting.Scope scope;
+    private final Setting.Property scope;
     private static final Pattern KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])*[-\\w]+$");
     private static final Pattern GROUP_KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])+$");
 
-    protected AbstractScopedSettings(Settings settings, Set<Setting<?>> settingsSet, Setting.Scope scope) {
+    protected AbstractScopedSettings(Settings settings, Set<Setting<?>> settingsSet, Setting.Property scope) {
         super(settings);
         this.lastSettingsApplied = Settings.EMPTY;
         this.scope = scope;
         Map<String, Setting<?>> complexMatchers = new HashMap<>();
         Map<String, Setting<?>> keySettings = new HashMap<>();
         for (Setting<?> setting : settingsSet) {
-            if (setting.getScope() != scope) {
-                throw new IllegalArgumentException("Setting must be a " + scope + " setting but was: " + setting.getScope());
-            }
-            if (isValidKey(setting.getKey()) == false && (setting.isGroupSetting() && isValidGroupKey(setting.getKey())) == false) {
-                throw new IllegalArgumentException("illegal settings key: [" + setting.getKey() + "]");
+            if (setting.getProperties().contains(scope) == false) {
+                throw new IllegalArgumentException("Setting must be a " + scope + " setting but has: " + setting.getProperties());
             }
+            validateSettingKey(setting);
+
             if (setting.hasComplexMatcher()) {
                 Setting<?> overlappingSetting = findOverlappingSetting(setting, complexMatchers);
                 if (overlappingSetting != null) {
@@ -76,6 +84,12 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
         this.keySettings = Collections.unmodifiableMap(keySettings);
     }
 
+    protected void validateSettingKey(Setting setting) {
+        if (isValidKey(setting.getKey()) == false && (setting.isGroupSetting() && isValidGroupKey(setting.getKey())) == false) {
+            throw new IllegalArgumentException("illegal settings key: [" + setting.getKey() + "]");
+        }
+    }
+
     protected AbstractScopedSettings(Settings nodeSettings, Settings scopeSettings, AbstractScopedSettings other) {
         super(nodeSettings);
         this.lastSettingsApplied = scopeSettings;
@@ -96,7 +110,7 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
         return GROUP_KEY_PATTERN.matcher(key).matches();
     }
 
-    public Setting.Scope getScope() {
+    public Setting.Property getScope() {
         return this.scope;
     }
 
@@ -216,9 +230,17 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
      * * Validates that all given settings are registered and valid
      */
     public final void validate(Settings settings) {
-        for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
-            validate(entry.getKey(), settings);
+        List<RuntimeException> exceptions = new ArrayList<>();
+        // we want them sorted for deterministic error messages
+        SortedMap<String, String> sortedSettings = new TreeMap<>(settings.getAsMap());
+        for (Map.Entry<String, String> entry : sortedSettings.entrySet()) {
+            try {
+                validate(entry.getKey(), settings);
+            } catch (RuntimeException ex) {
+                exceptions.add(ex);
+            }
         }
+        ExceptionsHelper.rethrowAndSuppress(exceptions);
     }
 
 
@@ -228,7 +250,21 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
     public final void validate(String key, Settings settings) {
         Setting setting = get(key);
         if (setting == null) {
-            throw new IllegalArgumentException("unknown setting [" + key + "]");
+            LevensteinDistance ld = new LevensteinDistance();
+            List<Tuple<Float, String>> scoredKeys = new ArrayList<>();
+            for (String k : this.keySettings.keySet()) {
+                float distance = ld.getDistance(key, k);
+                if (distance > 0.7f) {
+                    scoredKeys.add(new Tuple<>(distance, k));
+                }
+            }
+            CollectionUtil.timSort(scoredKeys, (a,b) -> b.v1().compareTo(a.v1()));
+            String msg = "unknown setting [" + key + "]";
+            List<String> keys = scoredKeys.stream().map((a) -> a.v2()).collect(Collectors.toList());
+            if (keys.isEmpty() == false) {
+                msg += " did you mean " + (keys.size() == 1 ? "[" + keys.get(0) + "]": "any of " + keys.toString()) + "?";
+            }
+            throw new IllegalArgumentException(msg);
         }
         setting.get(settings);
     }
@@ -342,8 +378,9 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
      * Returns the value for the given setting.
      */
     public <T> T get(Setting<T> setting) {
-        if (setting.getScope() != scope) {
-            throw new IllegalArgumentException("settings scope doesn't match the setting scope [" + this.scope + "] != [" + setting.getScope() + "]");
+        if (setting.getProperties().contains(scope) == false) {
+            throw new IllegalArgumentException("settings scope doesn't match the setting scope [" + this.scope + "] not in [" +
+                setting.getProperties() + "]");
         }
         if (get(setting.getKey()) == null) {
             throw new IllegalArgumentException("setting " + setting.getKey() + " has not been registered");
@@ -442,4 +479,53 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
         }
         return null;
     }
+
+    /**
+     * Archives broken or unknown settings. Any setting that is not recognized or fails
+     * validation will be archived. This means the setting is prefixed with {@value ARCHIVED_SETTINGS_PREFIX}
+     * and remains in the settings object. This can be used to detect broken settings via APIs.
+     */
+    public Settings archiveUnknownOrBrokenSettings(Settings settings) {
+        Settings.Builder builder = Settings.builder();
+        boolean changed = false;
+        for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
+            try {
+                Setting<?> setting = get(entry.getKey());
+                if (setting != null) {
+                    setting.get(settings);
+                    builder.put(entry.getKey(), entry.getValue());
+                } else {
+                    if (entry.getKey().startsWith(ARCHIVED_SETTINGS_PREFIX) || isPrivateSetting(entry.getKey())) {
+                        builder.put(entry.getKey(), entry.getValue());
+                    } else {
+                        changed = true;
+                        logger.warn("found unknown setting: {} value: {} - archiving", entry.getKey(), entry.getValue());
+                        // we put them back in here such that tools can check from the outside if there are any indices with broken settings. The setting can remain there
+                        // but we want users to be aware that some of their setting are broken and they can research why and what they need to do to replace them.
+                        builder.put(ARCHIVED_SETTINGS_PREFIX + entry.getKey(), entry.getValue());
+                    }
+                }
+            } catch (IllegalArgumentException ex) {
+                changed = true;
+                logger.warn("found invalid setting: {} value: {} - archiving",ex , entry.getKey(), entry.getValue());
+                // we put them back in here such that tools can check from the outside if there are any indices with broken settings. The setting can remain there
+                // but we want users to be aware that some of their setting sare broken and they can research why and what they need to do to replace them.
+                builder.put(ARCHIVED_SETTINGS_PREFIX + entry.getKey(), entry.getValue());
+            }
+        }
+        if (changed) {
+            return builder.build();
+        } else {
+            return settings;
+        }
+    }
+
+    /**
+     * Returns <code>true</code> iff the setting is a private setting ie. it should be treated as valid even though it has no internal
+     * representation. Otherwise <code>false</code>
+     */
+    // TODO this should be replaced by Setting.Property.HIDDEN or something like this.
+    protected boolean isPrivateSetting(String key) {
+        return false;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index fa8b8c4ac41..5f587cc270d 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -29,8 +29,10 @@ import org.elasticsearch.client.transport.TransportClientNodesService;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.InternalClusterInfoService;
+import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
@@ -41,10 +43,11 @@ import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDeci
 import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
-import org.elasticsearch.cluster.service.InternalClusterService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.discovery.DiscoveryModule;
@@ -101,7 +104,7 @@ import java.util.function.Predicate;
  */
 public final class ClusterSettings extends AbstractScopedSettings {
     public ClusterSettings(Settings nodeSettings, Set<Setting<?>> settingsSet) {
-        super(nodeSettings, settingsSet, Setting.Scope.CLUSTER);
+        super(nodeSettings, settingsSet, Property.NodeScope);
         addSettingsUpdater(new LoggingSettingUpdater(nodeSettings));
     }
 
@@ -252,14 +255,14 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING,
                     HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING,
                     HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING,
-                    InternalClusterService.CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING,
+                    ClusterService.CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING,
                     SearchService.DEFAULT_SEARCH_TIMEOUT_SETTING,
                     ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING,
                     TransportService.TRACE_LOG_EXCLUDE_SETTING,
                     TransportService.TRACE_LOG_INCLUDE_SETTING,
                     TransportCloseIndexAction.CLUSTER_INDICES_CLOSE_ENABLE_SETTING,
                     ShardsLimitAllocationDecider.CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING,
-                    InternalClusterService.CLUSTER_SERVICE_RECONNECT_INTERVAL_SETTING,
+                    NodeConnectionsService.CLUSTER_NODE_RECONNECT_INTERVAL_SETTING,
                     HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_TYPE_SETTING,
                     HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_TYPE_SETTING,
                     Transport.TRANSPORT_TCP_COMPRESS,
@@ -326,7 +329,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     Environment.PATH_SCRIPTS_SETTING,
                     Environment.PATH_SHARED_DATA_SETTING,
                     Environment.PIDFILE_SETTING,
-                    InternalClusterService.NODE_ID_SEED_SETTING,
+                    DiscoveryNodeService.NODE_ID_SEED_SETTING,
                     DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING,
                     DiscoveryModule.DISCOVERY_TYPE_SETTING,
                     DiscoveryModule.ZEN_MASTER_SERVICE_TYPE_SETTING,
@@ -341,9 +344,8 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     ZenDiscovery.JOIN_RETRY_DELAY_SETTING,
                     ZenDiscovery.MAX_PINGS_FROM_ANOTHER_MASTER_SETTING,
                     ZenDiscovery.SEND_LEAVE_REQUEST_SETTING,
-                    ZenDiscovery.MASTER_ELECTION_FILTER_CLIENT_SETTING,
                     ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING,
-                    ZenDiscovery.MASTER_ELECTION_FILTER_DATA_SETTING,
+                    ZenDiscovery.MASTER_ELECTION_IGNORE_NON_MASTER_PINGS_SETTING,
                     UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING,
                     UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING,
                     SearchService.DEFAULT_KEEPALIVE_SETTING,
diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
index 69ef795812d..fb498283d7b 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -22,6 +22,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.gateway.PrimaryShardAllocator;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
@@ -34,7 +35,8 @@ import org.elasticsearch.index.engine.EngineConfig;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.percolator.PercolatorQueriesRegistry;
+import org.elasticsearch.index.percolator.PercolatorQueryCache;
+import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.FsDirectoryService;
 import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.index.store.Store;
@@ -44,12 +46,13 @@ import org.elasticsearch.indices.IndicesRequestCache;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 import java.util.function.Predicate;
 
 /**
  * Encapsulates all valid index level settings.
- * @see org.elasticsearch.common.settings.Setting.Scope#INDEX
+ * @see Property#IndexScope
  */
 public final class IndexScopedSettings extends AbstractScopedSettings {
 
@@ -122,7 +125,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         FieldMapper.IGNORE_MALFORMED_SETTING,
         FieldMapper.COERCE_SETTING,
         Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING,
-        PercolatorQueriesRegistry.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING,
+        PercolatorQueryCache.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING,
         MapperService.INDEX_MAPPER_DYNAMIC_SETTING,
         MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING,
         BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING,
@@ -132,17 +135,23 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         PrimaryShardAllocator.INDEX_RECOVERY_INITIAL_SHARDS_SETTING,
         FsDirectoryService.INDEX_LOCK_FACTOR_SETTING,
         EngineConfig.INDEX_CODEC_SETTING,
-        IndexWarmer.INDEX_NORMS_LOADING_SETTING,
-        // this sucks but we can't really validate all the analyzers/similarity in here
-        Setting.groupSetting("index.similarity.", false, Setting.Scope.INDEX), // this allows similarity settings to be passed
-        Setting.groupSetting("index.analysis.", false, Setting.Scope.INDEX) // this allows analysis settings to be passed
+        // validate that built-in similarities don't get redefined
+        Setting.groupSetting("index.similarity.", (s) -> {
+            Map<String, Settings> groups = s.getAsGroups();
+            for (String key : SimilarityService.BUILT_IN.keySet()) {
+                if (groups.containsKey(key)) {
+                    throw new IllegalArgumentException("illegal value for [index.similarity."+ key + "] cannot redefine built-in similarity");
+                }
+            }
+        }, Property.IndexScope), // this allows similarity settings to be passed
+        Setting.groupSetting("index.analysis.", Property.IndexScope) // this allows analysis settings to be passed
 
     )));
 
     public static final IndexScopedSettings DEFAULT_SCOPED_SETTINGS = new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
 
     public IndexScopedSettings(Settings settings, Set<Setting<?>> settingsSet) {
-        super(settings, settingsSet, Setting.Scope.INDEX);
+        super(settings, settingsSet, Property.IndexScope);
     }
 
     private IndexScopedSettings(Settings settings, IndexScopedSettings other, IndexMetaData metaData) {
@@ -153,7 +162,16 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         return new IndexScopedSettings(settings, this, metaData);
     }
 
-    public boolean isPrivateSetting(String key) {
+    @Override
+    protected void validateSettingKey(Setting setting) {
+        if (setting.getKey().startsWith("index.") == false) {
+            throw new IllegalArgumentException("illegal settings key: [" + setting.getKey() + "] must start with [index.]");
+        }
+        super.validateSettingKey(setting);
+    }
+
+    @Override
+    protected final boolean isPrivateSetting(String key) {
         switch (key) {
             case IndexMetaData.SETTING_CREATION_DATE:
             case IndexMetaData.SETTING_INDEX_UUID:
diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
index 0b4e43744a5..a2ebe7a2c30 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
@@ -20,37 +20,47 @@ package org.elasticsearch.common.settings;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.MemorySizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Enumeration;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 /**
  * A setting. Encapsulates typical stuff like default value, parsing, and scope.
- * Some (dynamic=true) can by modified at run time using the API.
+ * Some (SettingsProperty.Dynamic) can by modified at run time using the API.
  * All settings inside elasticsearch or in any of the plugins should use this type-safe and generic settings infrastructure
  * together with {@link AbstractScopedSettings}. This class contains several utility methods that makes it straight forward
  * to add settings for the majority of the cases. For instance a simple boolean settings can be defined like this:
  * <pre>{@code
- * public static final Setting<Boolean>; MY_BOOLEAN = Setting.boolSetting("my.bool.setting", true, false, Scope.CLUSTER);}
+ * public static final Setting<Boolean>; MY_BOOLEAN = Setting.boolSetting("my.bool.setting", true, SettingsProperty.NodeScope);}
  * </pre>
  * To retrieve the value of the setting a {@link Settings} object can be passed directly to the {@link Setting#get(Settings)} method.
  * <pre>
@@ -61,32 +71,81 @@ import java.util.stream.Collectors;
  * public enum Color {
  *     RED, GREEN, BLUE;
  * }
- * public static final Setting<Color> MY_BOOLEAN = new Setting<>("my.color.setting", Color.RED.toString(), Color::valueOf, false, Scope.CLUSTER);
+ * public static final Setting<Color> MY_BOOLEAN =
+ *     new Setting<>("my.color.setting", Color.RED.toString(), Color::valueOf, SettingsProperty.NodeScope);
  * }
  * </pre>
  */
 public class Setting<T> extends ToXContentToBytes {
+
+    public enum Property {
+        /**
+         * should be filtered in some api (mask password/credentials)
+         */
+        Filtered,
+
+        /**
+         * iff this setting can be dynamically updateable
+         */
+        Dynamic,
+
+        /**
+         * mark this setting as deprecated
+         */
+        Deprecated,
+
+        /**
+         * Node scope
+         */
+        NodeScope,
+
+        /**
+         * Index scope
+         */
+        IndexScope
+    }
+
+    private static final ESLogger logger = Loggers.getLogger(Setting.class);
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
+
     private final Key key;
     protected final Function<Settings, String> defaultValue;
     private final Function<String, T> parser;
-    private final boolean dynamic;
-    private final Scope scope;
+    private final EnumSet<Property> properties;
+
+    private static final EnumSet<Property> EMPTY_PROPERTIES = EnumSet.noneOf(Property.class);
 
     /**
-     * Creates a new Setting instance
+     * Creates a new Setting instance. When no scope is provided, we default to {@link Property#NodeScope}.
      * @param key the settings key for this setting.
      * @param defaultValue a default value function that returns the default values string representation.
      * @param parser a parser that parses the string rep into a complex datatype.
-     * @param dynamic true iff this setting can be dynamically updateable
-     * @param scope the scope of this setting
+     * @param properties properties for this setting like scope, filtering...
      */
-    public Setting(Key key, Function<Settings, String> defaultValue, Function<String, T> parser, boolean dynamic, Scope scope) {
+    public Setting(Key key, Function<Settings, String> defaultValue, Function<String, T> parser, Property... properties) {
         assert parser.apply(defaultValue.apply(Settings.EMPTY)) != null || this.isGroupSetting(): "parser returned null";
         this.key = key;
         this.defaultValue = defaultValue;
         this.parser = parser;
-        this.dynamic = dynamic;
-        this.scope = scope;
+        if (properties == null) {
+            throw new IllegalArgumentException("properties can not be null for setting [" + key + "]");
+        }
+        if (properties.length == 0) {
+            this.properties = EMPTY_PROPERTIES;
+        } else {
+            this.properties = EnumSet.copyOf(Arrays.asList(properties));
+        }
+    }
+
+    /**
+     * Creates a new Setting instance
+     * @param key the settings key for this setting.
+     * @param defaultValue a default value.
+     * @param parser a parser that parses the string rep into a complex datatype.
+     * @param properties properties for this setting like scope, filtering...
+     */
+    public Setting(String key, String defaultValue, Function<String, T> parser, Property... properties) {
+        this(key, s -> defaultValue, parser, properties);
     }
 
     /**
@@ -94,11 +153,10 @@ public class Setting<T> extends ToXContentToBytes {
      * @param key the settings key for this setting.
      * @param defaultValue a default value function that returns the default values string representation.
      * @param parser a parser that parses the string rep into a complex datatype.
-     * @param dynamic true iff this setting can be dynamically updateable
-     * @param scope the scope of this setting
+     * @param properties properties for this setting like scope, filtering...
      */
-    public Setting(String key, Function<Settings, String> defaultValue, Function<String, T> parser, boolean dynamic, Scope scope) {
-        this(new SimpleKey(key), defaultValue, parser, dynamic, scope);
+    public Setting(String key, Function<Settings, String> defaultValue, Function<String, T> parser, Property... properties) {
+        this(new SimpleKey(key), defaultValue, parser, properties);
     }
 
     /**
@@ -106,11 +164,10 @@ public class Setting<T> extends ToXContentToBytes {
      * @param key the settings key for this setting.
      * @param fallBackSetting a setting to fall back to if the current setting is not set.
      * @param parser a parser that parses the string rep into a complex datatype.
-     * @param dynamic true iff this setting can be dynamically updateable
-     * @param scope the scope of this setting
+     * @param properties properties for this setting like scope, filtering...
      */
-    public Setting(String key, Setting<T> fallBackSetting, Function<String, T> parser, boolean dynamic, Scope scope) {
-        this(key, fallBackSetting::getRaw, parser, dynamic, scope);
+    public Setting(String key, Setting<T> fallBackSetting, Function<String, T> parser, Property... properties) {
+        this(key, fallBackSetting::getRaw, parser, properties);
     }
 
     /**
@@ -132,17 +189,46 @@ public class Setting<T> extends ToXContentToBytes {
     }
 
     /**
-     * Returns <code>true</code> iff this setting is dynamically updateable, otherwise <code>false</code>
+     * Returns <code>true</code> if this setting is dynamically updateable, otherwise <code>false</code>
      */
     public final boolean isDynamic() {
-        return dynamic;
+        return properties.contains(Property.Dynamic);
     }
 
     /**
-     * Returns the settings scope
+     * Returns the setting properties
+     * @see Property
      */
-    public final Scope getScope() {
-        return scope;
+    public EnumSet<Property> getProperties() {
+        return properties;
+    }
+
+    /**
+     * Returns <code>true</code> if this setting must be filtered, otherwise <code>false</code>
+     */
+    public boolean isFiltered() {
+        return properties.contains(Property.Filtered);
+    }
+
+    /**
+     * Returns <code>true</code> if this setting has a node scope, otherwise <code>false</code>
+     */
+    public boolean hasNodeScope() {
+        return properties.contains(Property.NodeScope);
+    }
+
+    /**
+     * Returns <code>true</code> if this setting has an index scope, otherwise <code>false</code>
+     */
+    public boolean hasIndexScope() {
+        return properties.contains(Property.IndexScope);
+    }
+
+    /**
+     * Returns <code>true</code> if this setting is deprecated, otherwise <code>false</code>
+     */
+    public boolean isDeprecated() {
+        return properties.contains(Property.Deprecated);
     }
 
     /**
@@ -177,7 +263,7 @@ public class Setting<T> extends ToXContentToBytes {
     /**
      * Returns <code>true</code> iff this setting is present in the given settings object. Otherwise <code>false</code>
      */
-    public final boolean exists(Settings settings) {
+    public boolean exists(Settings settings) {
         return settings.get(getKey()) != null;
     }
 
@@ -205,6 +291,12 @@ public class Setting<T> extends ToXContentToBytes {
      * instead. This is useful if the value can't be parsed due to an invalid value to access the actual value.
      */
     public String getRaw(Settings settings) {
+        // They're using the setting, so we need to tell them to stop
+        if (this.isDeprecated() && this.exists(settings)) {
+            // It would be convenient to show its replacement key, but replacement is often not so simple
+            deprecationLogger.deprecated("[{}] setting was deprecated in Elasticsearch and it will be removed in a future release! " +
+                    "See the breaking changes lists in the documentation for details", getKey());
+        }
         return settings.get(getKey(), defaultValue.apply(settings));
     }
 
@@ -221,8 +313,7 @@ public class Setting<T> extends ToXContentToBytes {
     public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field("key", key.toString());
-        builder.field("type", scope.name());
-        builder.field("dynamic", dynamic);
+        builder.field("properties", properties);
         builder.field("is_group_setting", isGroupSetting());
         builder.field("default", defaultValue.apply(Settings.EMPTY));
         builder.endObject();
@@ -244,14 +335,6 @@ public class Setting<T> extends ToXContentToBytes {
         return this;
     }
 
-    /**
-     * The settings scope - settings can either be cluster settings or per index settings.
-     */
-    public enum Scope {
-        CLUSTER,
-        INDEX;
-    }
-
     /**
      * Build a new updater with a noop validator.
      */
@@ -349,38 +432,34 @@ public class Setting<T> extends ToXContentToBytes {
     }
 
 
-    public Setting(String key, String defaultValue, Function<String, T> parser, boolean dynamic, Scope scope) {
-        this(key, (s) -> defaultValue, parser, dynamic, scope);
+    public static Setting<Float> floatSetting(String key, float defaultValue, Property... properties) {
+        return new Setting<>(key, (s) -> Float.toString(defaultValue), Float::parseFloat, properties);
     }
 
-    public static Setting<Float> floatSetting(String key, float defaultValue, boolean dynamic, Scope scope) {
-        return new Setting<>(key, (s) -> Float.toString(defaultValue), Float::parseFloat, dynamic, scope);
-    }
-
-    public static Setting<Float> floatSetting(String key, float defaultValue, float minValue, boolean dynamic, Scope scope) {
+    public static Setting<Float> floatSetting(String key, float defaultValue, float minValue, Property... properties) {
         return new Setting<>(key, (s) -> Float.toString(defaultValue), (s) -> {
             float value = Float.parseFloat(s);
             if (value < minValue) {
                 throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be >= " + minValue);
             }
             return value;
-        }, dynamic, scope);
+        }, properties);
     }
 
-    public static Setting<Integer> intSetting(String key, int defaultValue, int minValue, int maxValue, boolean dynamic, Scope scope) {
-        return new Setting<>(key, (s) -> Integer.toString(defaultValue), (s) -> parseInt(s, minValue, maxValue, key), dynamic, scope);
+    public static Setting<Integer> intSetting(String key, int defaultValue, int minValue, int maxValue, Property... properties) {
+        return new Setting<>(key, (s) -> Integer.toString(defaultValue), (s) -> parseInt(s, minValue, maxValue, key), properties);
     }
 
-    public static Setting<Integer> intSetting(String key, int defaultValue, int minValue, boolean dynamic, Scope scope) {
-        return new Setting<>(key, (s) -> Integer.toString(defaultValue), (s) -> parseInt(s, minValue, key), dynamic, scope);
+    public static Setting<Integer> intSetting(String key, int defaultValue, int minValue, Property... properties) {
+        return new Setting<>(key, (s) -> Integer.toString(defaultValue), (s) -> parseInt(s, minValue, key), properties);
     }
 
-    public static Setting<Long> longSetting(String key, long defaultValue, long minValue, boolean dynamic, Scope scope) {
-        return new Setting<>(key, (s) -> Long.toString(defaultValue), (s) -> parseLong(s, minValue, key), dynamic, scope);
+    public static Setting<Long> longSetting(String key, long defaultValue, long minValue, Property... properties) {
+        return new Setting<>(key, (s) -> Long.toString(defaultValue), (s) -> parseLong(s, minValue, key), properties);
     }
 
-    public static Setting<String> simpleString(String key, boolean dynamic, Scope scope) {
-        return new Setting<>(key, "", Function.identity(), dynamic, scope);
+    public static Setting<String> simpleString(String key, Property... properties) {
+        return new Setting<>(key, s -> "", Function.identity(), properties);
     }
 
     public static int parseInt(String s, int minValue, String key) {
@@ -414,51 +493,80 @@ public class Setting<T> extends ToXContentToBytes {
         return timeValue;
     }
 
-    public static Setting<Integer> intSetting(String key, int defaultValue, boolean dynamic, Scope scope) {
-        return intSetting(key, defaultValue, Integer.MIN_VALUE, dynamic, scope);
+    public static Setting<Integer> intSetting(String key, int defaultValue, Property... properties) {
+        return intSetting(key, defaultValue, Integer.MIN_VALUE, properties);
     }
 
-    public static Setting<Boolean> boolSetting(String key, boolean defaultValue, boolean dynamic, Scope scope) {
-        return new Setting<>(key, (s) -> Boolean.toString(defaultValue), Booleans::parseBooleanExact, dynamic, scope);
+    public static Setting<Boolean> boolSetting(String key, boolean defaultValue, Property... properties) {
+        return new Setting<>(key, (s) -> Boolean.toString(defaultValue), Booleans::parseBooleanExact, properties);
     }
 
-    public static Setting<Boolean> boolSetting(String key, Setting<Boolean> fallbackSetting, boolean dynamic, Scope scope) {
-        return new Setting<>(key, fallbackSetting, Booleans::parseBooleanExact, dynamic, scope);
+    public static Setting<Boolean> boolSetting(String key, Setting<Boolean> fallbackSetting, Property... properties) {
+        return new Setting<>(key, fallbackSetting, Booleans::parseBooleanExact, properties);
     }
 
-    public static Setting<ByteSizeValue> byteSizeSetting(String key, String percentage, boolean dynamic, Scope scope) {
-        return new Setting<>(key, (s) -> percentage, (s) -> MemorySizeValue.parseBytesSizeValueOrHeapRatio(s, key), dynamic, scope);
+    public static Setting<ByteSizeValue> byteSizeSetting(String key, String percentage, Property... properties) {
+        return new Setting<>(key, (s) -> percentage, (s) -> MemorySizeValue.parseBytesSizeValueOrHeapRatio(s, key), properties);
     }
 
-    public static Setting<ByteSizeValue> byteSizeSetting(String key, ByteSizeValue value, boolean dynamic, Scope scope) {
-        return byteSizeSetting(key, (s) -> value.toString(), dynamic, scope);
+    public static Setting<ByteSizeValue> byteSizeSetting(String key, ByteSizeValue value, Property... properties) {
+        return byteSizeSetting(key, (s) -> value.toString(), properties);
     }
 
-    public static Setting<ByteSizeValue> byteSizeSetting(String key, Setting<ByteSizeValue> fallbackSettings, boolean dynamic, Scope scope) {
-        return byteSizeSetting(key, fallbackSettings::getRaw, dynamic, scope);
+    public static Setting<ByteSizeValue> byteSizeSetting(String key, Setting<ByteSizeValue> fallbackSettings,
+                                                         Property... properties) {
+        return byteSizeSetting(key, fallbackSettings::getRaw, properties);
     }
 
-    public static Setting<ByteSizeValue> byteSizeSetting(String key, Function<Settings, String> defaultValue, boolean dynamic, Scope scope) {
-        return new Setting<>(key, defaultValue, (s) -> ByteSizeValue.parseBytesSizeValue(s, key), dynamic, scope);
+    public static Setting<ByteSizeValue> byteSizeSetting(String key, Function<Settings, String> defaultValue,
+                                                         Property... properties) {
+        return new Setting<>(key, defaultValue, (s) -> ByteSizeValue.parseBytesSizeValue(s, key), properties);
     }
 
-    public static Setting<TimeValue> positiveTimeSetting(String key, TimeValue defaultValue, boolean dynamic, Scope scope) {
-        return timeSetting(key, defaultValue, TimeValue.timeValueMillis(0), dynamic, scope);
+    public static Setting<ByteSizeValue> byteSizeSetting(String key, ByteSizeValue value, ByteSizeValue minValue, ByteSizeValue maxValue,
+                                                         Property... properties) {
+        return byteSizeSetting(key, (s) -> value.toString(), minValue, maxValue, properties);
     }
 
-    public static <T> Setting<List<T>> listSetting(String key, List<String> defaultStringValue, Function<String, T> singleValueParser, boolean dynamic, Scope scope) {
-        return listSetting(key, (s) -> defaultStringValue, singleValueParser, dynamic, scope);
+    public static Setting<ByteSizeValue> byteSizeSetting(String key, Function<Settings, String> defaultValue,
+                                                         ByteSizeValue minValue, ByteSizeValue maxValue,
+                                                         Property... properties) {
+        return new Setting<>(key, defaultValue, (s) -> parseByteSize(s, minValue, maxValue, key), properties);
     }
 
-    public static <T> Setting<List<T>> listSetting(String key, Setting<List<T>> fallbackSetting, Function<String, T> singleValueParser, boolean dynamic, Scope scope) {
-        return listSetting(key, (s) -> parseableStringToList(fallbackSetting.getRaw(s)), singleValueParser, dynamic, scope);
+    public static ByteSizeValue parseByteSize(String s, ByteSizeValue minValue, ByteSizeValue maxValue, String key) {
+        ByteSizeValue value = ByteSizeValue.parseBytesSizeValue(s, key);
+        if (value.bytes() < minValue.bytes()) {
+            throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be >= " + minValue);
+        }
+        if (value.bytes() > maxValue.bytes()) {
+            throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be =< " + maxValue);
+        }
+        return value;
     }
 
-    public static <T> Setting<List<T>> listSetting(String key, Function<Settings, List<String>> defaultStringValue, Function<String, T> singleValueParser, boolean dynamic, Scope scope) {
+    public static Setting<TimeValue> positiveTimeSetting(String key, TimeValue defaultValue, Property... properties) {
+        return timeSetting(key, defaultValue, TimeValue.timeValueMillis(0), properties);
+    }
+
+    public static <T> Setting<List<T>> listSetting(String key, List<String> defaultStringValue, Function<String, T> singleValueParser,
+                                                   Property... properties) {
+        return listSetting(key, (s) -> defaultStringValue, singleValueParser, properties);
+    }
+
+    public static <T> Setting<List<T>> listSetting(String key, Setting<List<T>> fallbackSetting, Function<String, T> singleValueParser,
+                                                   Property... properties) {
+        return listSetting(key, (s) -> parseableStringToList(fallbackSetting.getRaw(s)), singleValueParser, properties);
+    }
+
+    public static <T> Setting<List<T>> listSetting(String key, Function<Settings, List<String>> defaultStringValue,
+                                                   Function<String, T> singleValueParser, Property... properties) {
         Function<String, List<T>> parser = (s) ->
                 parseableStringToList(s).stream().map(singleValueParser).collect(Collectors.toList());
 
-        return new Setting<List<T>>(new ListKey(key), (s) -> arrayToParsableString(defaultStringValue.apply(s).toArray(Strings.EMPTY_ARRAY)), parser, dynamic, scope) {
+        return new Setting<List<T>>(new ListKey(key),
+            (s) -> arrayToParsableString(defaultStringValue.apply(s).toArray(Strings.EMPTY_ARRAY)), parser, properties) {
+            private final Pattern pattern = Pattern.compile(Pattern.quote(key)+"(\\.\\d+)?");
             @Override
             public String getRaw(Settings settings) {
                 String[] array = settings.getAsArray(getKey(), null);
@@ -505,17 +613,45 @@ public class Setting<T> extends ToXContentToBytes {
             throw new ElasticsearchException(ex);
         }
     }
-
-    public static Setting<Settings> groupSetting(String key, boolean dynamic, Scope scope) {
-        return new Setting<Settings>(new GroupKey(key), (s) -> "", (s) -> null, dynamic, scope) {
+    public static Setting<Settings> groupSetting(String key, Property... properties) {
+        return groupSetting(key, (s) -> {}, properties);
+    }
+    public static Setting<Settings> groupSetting(String key, Consumer<Settings> validator, Property... properties) {
+        return new Setting<Settings>(new GroupKey(key), (s) -> "", (s) -> null, properties) {
             @Override
             public boolean isGroupSetting() {
                 return true;
             }
 
+            @Override
+            public String getRaw(Settings settings) {
+                Settings subSettings = get(settings);
+                try {
+                    XContentBuilder builder = XContentFactory.jsonBuilder();
+                    builder.startObject();
+                    subSettings.toXContent(builder, EMPTY_PARAMS);
+                    builder.endObject();
+                    return builder.string();
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+
             @Override
             public Settings get(Settings settings) {
-                return settings.getByPrefix(getKey());
+                Settings byPrefix = settings.getByPrefix(getKey());
+                validator.accept(byPrefix);
+                return byPrefix;
+            }
+
+            @Override
+            public boolean exists(Settings settings) {
+                for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
+                    if (entry.getKey().startsWith(key)) {
+                        return true;
+                    }
+                }
+                return false;
             }
 
             @Override
@@ -560,30 +696,37 @@ public class Setting<T> extends ToXContentToBytes {
         };
     }
 
-    public static Setting<TimeValue> timeSetting(String key, Function<Settings, String> defaultValue, TimeValue minValue, boolean dynamic, Scope scope) {
-        return new Setting<>(key, defaultValue, (s) -> parseTimeValue(s, minValue, key), dynamic, scope);
+    public static Setting<TimeValue> timeSetting(String key, Function<Settings, String> defaultValue, TimeValue minValue,
+                                                 Property... properties) {
+        return new Setting<>(key, defaultValue, (s) -> {
+            TimeValue timeValue = TimeValue.parseTimeValue(s, null, key);
+            if (timeValue.millis() < minValue.millis()) {
+                throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be >= " + minValue);
+            }
+            return timeValue;
+        }, properties);
     }
 
-    public static Setting<TimeValue> timeSetting(String key, TimeValue defaultValue, TimeValue minValue, boolean dynamic, Scope scope) {
-        return timeSetting(key, (s) -> defaultValue.getStringRep(), minValue, dynamic, scope);
+    public static Setting<TimeValue> timeSetting(String key, TimeValue defaultValue, TimeValue minValue, Property... properties) {
+        return timeSetting(key, (s) -> defaultValue.getStringRep(), minValue, properties);
     }
 
-    public static Setting<TimeValue> timeSetting(String key, TimeValue defaultValue, boolean dynamic, Scope scope) {
-        return new Setting<>(key, (s) -> defaultValue.toString(), (s) -> TimeValue.parseTimeValue(s, key), dynamic, scope);
+    public static Setting<TimeValue> timeSetting(String key, TimeValue defaultValue, Property... properties) {
+        return new Setting<>(key, (s) -> defaultValue.toString(), (s) -> TimeValue.parseTimeValue(s, key), properties);
     }
 
-    public static Setting<TimeValue> timeSetting(String key, Setting<TimeValue> fallbackSetting, boolean dynamic, Scope scope) {
-        return new Setting<>(key, fallbackSetting::getRaw, (s) -> TimeValue.parseTimeValue(s, key), dynamic, scope);
+    public static Setting<TimeValue> timeSetting(String key, Setting<TimeValue> fallbackSetting, Property... properties) {
+        return new Setting<>(key, fallbackSetting::getRaw, (s) -> TimeValue.parseTimeValue(s, key), properties);
     }
 
-    public static Setting<Double> doubleSetting(String key, double defaultValue, double minValue, boolean dynamic, Scope scope) {
+    public static Setting<Double> doubleSetting(String key, double defaultValue, double minValue, Property... properties) {
         return new Setting<>(key, (s) -> Double.toString(defaultValue), (s) -> {
             final double d = Double.parseDouble(s);
             if (d < minValue) {
                 throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be >= " + minValue);
             }
             return d;
-        }, dynamic, scope);
+        }, properties);
     }
 
     @Override
@@ -604,8 +747,9 @@ public class Setting<T> extends ToXContentToBytes {
      * can easily be added with this setting. Yet, prefix key settings don't support updaters out of the box unless
      * {@link #getConcreteSetting(String)} is used to pull the updater.
      */
-    public static <T> Setting<T> prefixKeySetting(String prefix, String defaultValue, Function<String, T> parser, boolean dynamic, Scope scope) {
-        return affixKeySetting(AffixKey.withPrefix(prefix), (s) -> defaultValue, parser, dynamic, scope);
+    public static <T> Setting<T> prefixKeySetting(String prefix, String defaultValue, Function<String, T> parser,
+                                                  Property... properties) {
+        return affixKeySetting(AffixKey.withPrefix(prefix), (s) -> defaultValue, parser, properties);
     }
 
     /**
@@ -613,16 +757,19 @@ public class Setting<T> extends ToXContentToBytes {
      * storage.${backend}.enable=[true|false] can easily be added with this setting. Yet, adfix key settings don't support updaters
      * out of the box unless {@link #getConcreteSetting(String)} is used to pull the updater.
      */
-    public static <T> Setting<T> adfixKeySetting(String prefix, String suffix, Function<Settings, String> defaultValue, Function<String, T> parser, boolean dynamic, Scope scope) {
-        return affixKeySetting(AffixKey.withAdfix(prefix, suffix), defaultValue, parser, dynamic, scope);
+    public static <T> Setting<T> adfixKeySetting(String prefix, String suffix, Function<Settings, String> defaultValue,
+                                                 Function<String, T> parser, Property... properties) {
+        return affixKeySetting(AffixKey.withAdfix(prefix, suffix), defaultValue, parser, properties);
     }
 
-    public static <T> Setting<T> adfixKeySetting(String prefix, String suffix, String defaultValue, Function<String, T> parser, boolean dynamic, Scope scope) {
-        return adfixKeySetting(prefix, suffix, (s) -> defaultValue, parser, dynamic, scope);
+    public static <T> Setting<T> adfixKeySetting(String prefix, String suffix, String defaultValue, Function<String, T> parser,
+                                                 Property... properties) {
+        return adfixKeySetting(prefix, suffix, (s) -> defaultValue, parser, properties);
     }
 
-    public static <T> Setting<T> affixKeySetting(AffixKey key, Function<Settings, String> defaultValue, Function<String, T> parser, boolean dynamic, Scope scope) {
-        return new Setting<T>(key, defaultValue, parser, dynamic, scope) {
+    public static <T> Setting<T> affixKeySetting(AffixKey key, Function<Settings, String> defaultValue, Function<String, T> parser,
+                                                 Property... properties) {
+        return new Setting<T>(key, defaultValue, parser, properties) {
 
             @Override
             boolean isGroupSetting() {
@@ -637,7 +784,7 @@ public class Setting<T> extends ToXContentToBytes {
             @Override
             public Setting<T> getConcreteSetting(String key) {
                 if (match(key)) {
-                    return new Setting<>(key, defaultValue, parser, dynamic, scope);
+                    return new Setting<>(key, defaultValue, parser, properties);
                 } else {
                     throw new IllegalArgumentException("key [" + key + "] must match [" + getKey() + "] but didn't.");
                 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
index aafaff3e9d7..ce79bf92d20 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
@@ -761,6 +761,14 @@ public final class Settings implements ToXContent {
         return builder;
     }
 
+    /**
+     * Returns <tt>true</tt> if this settings object contains no settings
+     * @return <tt>true</tt> if this settings object contains no settings
+     */
+    public boolean isEmpty() {
+        return this.settings.isEmpty();
+    }
+
     /**
      * A builder allowing to put different settings and then {@link #build()} an immutable
      * settings implementation. Use {@link Settings#settingsBuilder()} in order to
@@ -1136,10 +1144,10 @@ public final class Settings implements ToXContent {
          * @param properties The properties to put
          * @return The builder
          */
-        public Builder putProperties(String prefix, Dictionary<Object,Object> properties) {
-            for (Object key1 : Collections.list(properties.keys())) {
-                String key = Objects.toString(key1);
-                String value = Objects.toString(properties.get(key));
+        public Builder putProperties(String prefix, Dictionary<Object, Object> properties) {
+            for (Object property : Collections.list(properties.keys())) {
+                String key = Objects.toString(property);
+                String value = Objects.toString(properties.get(property));
                 if (key.startsWith(prefix)) {
                     map.put(key.substring(prefix.length()), value);
                 }
@@ -1154,19 +1162,12 @@ public final class Settings implements ToXContent {
          * @param properties The properties to put
          * @return The builder
          */
-        public Builder putProperties(String prefix, Dictionary<Object,Object> properties, String[] ignorePrefixes) {
-            for (Object key1 : Collections.list(properties.keys())) {
-                String key = Objects.toString(key1);
-                String value = Objects.toString(properties.get(key));
+        public Builder putProperties(String prefix, Dictionary<Object, Object> properties, String ignorePrefix) {
+            for (Object property : Collections.list(properties.keys())) {
+                String key = Objects.toString(property);
+                String value = Objects.toString(properties.get(property));
                 if (key.startsWith(prefix)) {
-                    boolean ignore = false;
-                    for (String ignorePrefix : ignorePrefixes) {
-                        if (key.startsWith(ignorePrefix)) {
-                            ignore = true;
-                            break;
-                        }
-                    }
-                    if (!ignore) {
+                    if (!key.startsWith(ignorePrefix)) {
                         map.put(key.substring(prefix.length()), value);
                     }
                 }
@@ -1220,7 +1221,7 @@ public final class Settings implements ToXContent {
                     }
                 };
             for (Map.Entry<String, String> entry : new HashMap<>(map).entrySet()) {
-                String value = propertyPlaceholder.replacePlaceholders(entry.getValue(), placeholderResolver);
+                String value = propertyPlaceholder.replacePlaceholders(entry.getKey(), entry.getValue(), placeholderResolver);
                 // if the values exists and has length, we should maintain it  in the map
                 // otherwise, the replace process resolved into removing it
                 if (Strings.hasLength(value)) {
diff --git a/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java b/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
index b06f53459c8..2e7acd6ae8c 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
@@ -20,13 +20,22 @@
 package org.elasticsearch.common.settings;
 
 import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.tribe.TribeService;
 
+import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 /**
  * A module that binds the provided settings to the {@link Settings} interface.
@@ -35,11 +44,14 @@ public class SettingsModule extends AbstractModule {
 
     private final Settings settings;
     private final Set<String> settingsFilterPattern = new HashSet<>();
-    private final Map<String, Setting<?>> clusterSettings = new HashMap<>();
+    private final Map<String, Setting<?>> nodeSettings = new HashMap<>();
     private final Map<String, Setting<?>> indexSettings = new HashMap<>();
-    private static final Predicate<String> TRIBE_CLIENT_NODE_SETTINGS_PREDICATE =  (s) -> s.startsWith("tribe.") && TribeService.TRIBE_SETTING_KEYS.contains(s) == false;
+    private static final Predicate<String> TRIBE_CLIENT_NODE_SETTINGS_PREDICATE =  (s) -> s.startsWith("tribe.")
+        && TribeService.TRIBE_SETTING_KEYS.contains(s) == false;
+    private final ESLogger logger;
 
     public SettingsModule(Settings settings) {
+        logger = Loggers.getLogger(getClass(), settings);
         this.settings = settings;
         for (Setting<?> setting : ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) {
             registerSetting(setting);
@@ -52,10 +64,59 @@ public class SettingsModule extends AbstractModule {
     @Override
     protected void configure() {
         final IndexScopedSettings indexScopedSettings = new IndexScopedSettings(settings, new HashSet<>(this.indexSettings.values()));
-        final ClusterSettings clusterSettings = new ClusterSettings(settings, new HashSet<>(this.clusterSettings.values()));
+        final ClusterSettings clusterSettings = new ClusterSettings(settings, new HashSet<>(this.nodeSettings.values()));
+        Settings indexSettings = settings.filter((s) -> s.startsWith("index.") && clusterSettings.get(s) == null);
+        if (indexSettings.isEmpty() == false) {
+            try {
+                String separator = IntStream.range(0, 85).mapToObj(s -> "*").collect(Collectors.joining("")).trim();
+                StringBuilder builder = new StringBuilder();
+                builder.append(System.lineSeparator());
+                builder.append(separator);
+                builder.append(System.lineSeparator());
+                builder.append("Found index level settings on node level configuration.");
+                builder.append(System.lineSeparator());
+                builder.append(System.lineSeparator());
+                int count = 0;
+                for (String word : ("Since elasticsearch 5.x index level settings can NOT be set on the nodes configuration like " +
+                    "the elasticsearch.yaml, in system properties or command line arguments." +
+                    "In order to upgrade all indices the settings must be updated via the /${index}/_settings API. " +
+                    "Unless all settings are dynamic all indices must be closed in order to apply the upgrade" +
+                    "Indices created in the future should use index templates to set default values."
+                    ).split(" ")) {
+                    if (count + word.length() > 85) {
+                        builder.append(System.lineSeparator());
+                        count = 0;
+                    }
+                    count += word.length() + 1;
+                    builder.append(word).append(" ");
+                }
+
+                builder.append(System.lineSeparator());
+                builder.append(System.lineSeparator());
+                builder.append("Please ensure all required values are updated on all indices by executing: ");
+                builder.append(System.lineSeparator());
+                builder.append(System.lineSeparator());
+                builder.append("curl -XPUT 'http://localhost:9200/_all/_settings?preserve_existing=true' -d '");
+                try (XContentBuilder xContentBuilder = XContentBuilder.builder(XContentType.JSON.xContent())) {
+                    xContentBuilder.prettyPrint();
+                    xContentBuilder.startObject();
+                    indexSettings.toXContent(xContentBuilder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
+                    xContentBuilder.endObject();
+                    builder.append(xContentBuilder.string());
+                }
+                builder.append("'");
+                builder.append(System.lineSeparator());
+                builder.append(separator);
+                builder.append(System.lineSeparator());
+
+                logger.warn(builder.toString());
+                throw new IllegalArgumentException("node settings must not contain any index level settings");
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
         // by now we are fully configured, lets check node level settings for unregistered index settings
-        indexScopedSettings.validate(settings.filter(IndexScopedSettings.INDEX_SETTINGS_KEY_PREDICATE));
-        final Predicate<String> acceptOnlyClusterSettings = TRIBE_CLIENT_NODE_SETTINGS_PREDICATE.or(IndexScopedSettings.INDEX_SETTINGS_KEY_PREDICATE).negate();
+        final Predicate<String> acceptOnlyClusterSettings = TRIBE_CLIENT_NODE_SETTINGS_PREDICATE.negate();
         clusterSettings.validate(settings.filter(acceptOnlyClusterSettings));
         validateTribeSettings(settings, clusterSettings);
         bind(Settings.class).toInstance(settings);
@@ -71,19 +132,26 @@ public class SettingsModule extends AbstractModule {
      * the setting during startup.
      */
     public void registerSetting(Setting<?> setting) {
-        switch (setting.getScope()) {
-            case CLUSTER:
-                if (clusterSettings.containsKey(setting.getKey())) {
+        if (setting.isFiltered()) {
+            if (settingsFilterPattern.contains(setting.getKey()) == false) {
+                registerSettingsFilter(setting.getKey());
+            }
+        }
+        if (setting.hasNodeScope() || setting.hasIndexScope()) {
+            if (setting.hasNodeScope()) {
+                if (nodeSettings.containsKey(setting.getKey())) {
                     throw new IllegalArgumentException("Cannot register setting [" + setting.getKey() + "] twice");
                 }
-                clusterSettings.put(setting.getKey(), setting);
-                break;
-            case INDEX:
+                nodeSettings.put(setting.getKey(), setting);
+            }
+            if (setting.hasIndexScope()) {
                 if (indexSettings.containsKey(setting.getKey())) {
                     throw new IllegalArgumentException("Cannot register setting [" + setting.getKey() + "] twice");
                 }
                 indexSettings.put(setting.getKey(), setting);
-                break;
+            }
+        } else {
+            throw new IllegalArgumentException("No scope found for setting [" + setting.getKey() + "]");
         }
     }
 
@@ -101,21 +169,15 @@ public class SettingsModule extends AbstractModule {
         settingsFilterPattern.add(filter);
     }
 
-    public void registerSettingsFilterIfMissing(String filter) {
-        if (settingsFilterPattern.contains(filter) == false) {
-            registerSettingsFilter(filter);
-        }
-    }
-
     /**
      * Check if a setting has already been registered
      */
     public boolean exists(Setting<?> setting) {
-        switch (setting.getScope()) {
-            case CLUSTER:
-                return clusterSettings.containsKey(setting.getKey());
-            case INDEX:
-                return indexSettings.containsKey(setting.getKey());
+        if (setting.hasNodeScope()) {
+            return nodeSettings.containsKey(setting.getKey());
+        }
+        if (setting.hasIndexScope()) {
+            return indexSettings.containsKey(setting.getKey());
         }
         throw new IllegalArgumentException("setting scope is unknown. This should never happen!");
     }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/loader/JsonSettingsLoader.java b/core/src/main/java/org/elasticsearch/common/settings/loader/JsonSettingsLoader.java
index f6f77192c75..02f7a5c37a0 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/loader/JsonSettingsLoader.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/loader/JsonSettingsLoader.java
@@ -27,6 +27,10 @@ import org.elasticsearch.common.xcontent.XContentType;
  */
 public class JsonSettingsLoader extends XContentSettingsLoader {
 
+    public JsonSettingsLoader(boolean allowNullValues) {
+        super(allowNullValues);
+    }
+
     @Override
     public XContentType contentType() {
         return XContentType.JSON;
diff --git a/core/src/main/java/org/elasticsearch/common/settings/loader/PropertiesSettingsLoader.java b/core/src/main/java/org/elasticsearch/common/settings/loader/PropertiesSettingsLoader.java
index 57c9419f5b2..6ee1f58cf4b 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/loader/PropertiesSettingsLoader.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/loader/PropertiesSettingsLoader.java
@@ -24,10 +24,12 @@ import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.io.FastStringReader;
 import org.elasticsearch.common.io.stream.StreamInput;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
+import java.util.function.Supplier;
 
 /**
  * Settings loader that loads (parses) the settings in a properties format.
@@ -36,42 +38,49 @@ public class PropertiesSettingsLoader implements SettingsLoader {
 
     @Override
     public Map<String, String> load(String source) throws IOException {
-        Properties props = new NoDuplicatesProperties();
-        FastStringReader reader = new FastStringReader(source);
-        try {
-            props.load(reader);
-            Map<String, String> result = new HashMap<>();
-            for (Map.Entry entry : props.entrySet()) {
-                result.put((String) entry.getKey(), (String) entry.getValue());
-            }
-            return result;
-        } finally {
-            IOUtils.closeWhileHandlingException(reader);
-        }
+        return load(() -> new FastStringReader(source), (reader, props) -> props.load(reader));
     }
 
     @Override
     public Map<String, String> load(byte[] source) throws IOException {
-        Properties props = new NoDuplicatesProperties();
-        StreamInput stream = StreamInput.wrap(source);
+        return load(() -> StreamInput.wrap(source), (inStream, props) -> props.load(inStream));
+    }
+
+    private final <T extends Closeable> Map<String, String> load(
+            Supplier<T> supplier,
+            IOExceptionThrowingBiConsumer<T, Properties> properties
+    ) throws IOException {
+        T t = null;
         try {
-            props.load(stream);
-            Map<String, String> result = new HashMap<>();
+            t = supplier.get();
+            final Properties props = new NoDuplicatesProperties();
+            properties.accept(t, props);
+            final Map<String, String> result = new HashMap<>();
             for (Map.Entry entry : props.entrySet()) {
                 result.put((String) entry.getKey(), (String) entry.getValue());
             }
             return result;
         } finally {
-            IOUtils.closeWhileHandlingException(stream);
+            IOUtils.closeWhileHandlingException(t);
         }
     }
 
+    @FunctionalInterface
+    private interface IOExceptionThrowingBiConsumer<T, U> {
+        void accept(T t, U u) throws IOException;
+    }
+
     class NoDuplicatesProperties extends Properties {
         @Override
         public synchronized Object put(Object key, Object value) {
-            Object previousValue = super.put(key, value);
+            final Object previousValue = super.put(key, value);
             if (previousValue != null) {
-                throw new ElasticsearchParseException("duplicate settings key [{}] found, previous value [{}], current value [{}]", key, previousValue, value);
+                throw new ElasticsearchParseException(
+                        "duplicate settings key [{}] found, previous value [{}], current value [{}]",
+                        key,
+                        previousValue,
+                        value
+                );
             }
             return previousValue;
         }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java
index e55cb1092f2..5bf9916ee0e 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java
@@ -20,43 +20,63 @@
 package org.elasticsearch.common.settings.loader;
 
 /**
- * A settings loader factory automatically trying to identify what type of
- * {@link SettingsLoader} to use.
- *
- *
+ * A class holding factory methods for settings loaders that attempts
+ * to infer the type of the underlying settings content.
  */
 public final class SettingsLoaderFactory {
 
     private SettingsLoaderFactory() {
-
     }
 
     /**
-     * Returns a {@link SettingsLoader} based on the resource name.
+     * Returns a {@link SettingsLoader} based on the source resource
+     * name. This factory method assumes that if the resource name ends
+     * with ".json" then the content should be parsed as JSON, else if
+     * the resource name ends with ".yml" or ".yaml" then the content
+     * should be parsed as YAML, else if the resource name ends with
+     * ".properties" then the content should be parsed as properties,
+     * otherwise default to attempting to parse as JSON. Note that the
+     * parsers returned by this method will not accept null-valued
+     * keys.
+     *
+     * @param resourceName The resource name containing the settings
+     *                     content.
+     * @return A settings loader.
      */
     public static SettingsLoader loaderFromResource(String resourceName) {
         if (resourceName.endsWith(".json")) {
-            return new JsonSettingsLoader();
+            return new JsonSettingsLoader(false);
         } else if (resourceName.endsWith(".yml") || resourceName.endsWith(".yaml")) {
-            return new YamlSettingsLoader();
+            return new YamlSettingsLoader(false);
         } else if (resourceName.endsWith(".properties")) {
             return new PropertiesSettingsLoader();
         } else {
             // lets default to the json one
-            return new JsonSettingsLoader();
+            return new JsonSettingsLoader(false);
         }
     }
 
     /**
-     * Returns a {@link SettingsLoader} based on the actual settings source.
+     * Returns a {@link SettingsLoader} based on the source content.
+     * This factory method assumes that if the underlying content
+     * contains an opening and closing brace ('{' and '}') then the
+     * content should be parsed as JSON, else if the underlying content
+     * fails this condition but contains a ':' then the content should
+     * be parsed as YAML, and otherwise should be parsed as properties.
+     * Note that the JSON and YAML parsers returned by this method will
+     * accept null-valued keys.
+     *
+     * @param source The underlying settings content.
+     * @return A settings loader.
      */
     public static SettingsLoader loaderFromSource(String source) {
         if (source.indexOf('{') != -1 && source.indexOf('}') != -1) {
-            return new JsonSettingsLoader();
+            return new JsonSettingsLoader(true);
         }
         if (source.indexOf(':') != -1) {
-            return new YamlSettingsLoader();
+            return new YamlSettingsLoader(true);
         }
         return new PropertiesSettingsLoader();
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/loader/XContentSettingsLoader.java b/core/src/main/java/org/elasticsearch/common/settings/loader/XContentSettingsLoader.java
index 9c2f973b96e..3875c1ef85a 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/loader/XContentSettingsLoader.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/loader/XContentSettingsLoader.java
@@ -38,6 +38,12 @@ public abstract class XContentSettingsLoader implements SettingsLoader {
 
     public abstract XContentType contentType();
 
+    private final boolean allowNullValues;
+
+    XContentSettingsLoader(boolean allowNullValues) {
+        this.allowNullValues = allowNullValues;
+    }
+
     @Override
     public Map<String, String> load(String source) throws IOException {
         try (XContentParser parser = XContentFactory.xContent(contentType()).createParser(source)) {
@@ -153,6 +159,16 @@ public abstract class XContentSettingsLoader implements SettingsLoader {
                     currentValue
             );
         }
+
+        if (currentValue == null && !allowNullValues) {
+            throw new ElasticsearchParseException(
+                    "null-valued setting found for key [{}] found at line number [{}], column number [{}]",
+                    key,
+                    parser.getTokenLocation().lineNumber,
+                    parser.getTokenLocation().columnNumber
+            );
+        }
+
         settings.put(key, currentValue);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/loader/YamlSettingsLoader.java b/core/src/main/java/org/elasticsearch/common/settings/loader/YamlSettingsLoader.java
index 248fe090b5d..12cde976691 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/loader/YamlSettingsLoader.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/loader/YamlSettingsLoader.java
@@ -30,6 +30,10 @@ import java.util.Map;
  */
 public class YamlSettingsLoader extends XContentSettingsLoader {
 
+    public YamlSettingsLoader(boolean allowNullValues) {
+        super(allowNullValues);
+    }
+
     @Override
     public XContentType contentType() {
         return XContentType.YAML;
diff --git a/core/src/main/java/org/elasticsearch/common/unit/DistanceUnit.java b/core/src/main/java/org/elasticsearch/common/unit/DistanceUnit.java
index d0e91646c0f..b34c1101f9b 100644
--- a/core/src/main/java/org/elasticsearch/common/unit/DistanceUnit.java
+++ b/core/src/main/java/org/elasticsearch/common/unit/DistanceUnit.java
@@ -210,34 +210,6 @@ public enum DistanceUnit implements Writeable<DistanceUnit> {
         return defaultUnit;
     }
 
-    /**
-     * Write a {@link DistanceUnit} to a {@link StreamOutput}
-     * 
-     * @param out {@link StreamOutput} to write to
-     * @param unit {@link DistanceUnit} to write 
-     */
-    public static void writeDistanceUnit(StreamOutput out, DistanceUnit unit) throws IOException {
-        out.writeByte((byte) unit.ordinal());
-    }
-
-    /**
-     * Read a {@link DistanceUnit} from a {@link StreamInput} 
-     * 
-     * @param in {@link StreamInput} to read the {@link DistanceUnit} from
-     * @return {@link DistanceUnit} read from the {@link StreamInput}
-     * @throws IOException if no unit can be read from the {@link StreamInput}
-     * @throws IllegalArgumentException if no matching {@link DistanceUnit} can be found
-     */
-    public static DistanceUnit readDistanceUnit(StreamInput in) throws IOException {
-        byte b = in.readByte();
-
-        if(b<0 || b>=values().length) {
-            throw new IllegalArgumentException("No type for distance unit matching [" + b + "]");
-        } else {
-            return values()[b];
-        }
-    }
-
     /**
      * This class implements a value+unit tuple.
      */
@@ -324,23 +296,30 @@ public enum DistanceUnit implements Writeable<DistanceUnit> {
         }
     }
 
-    private static final DistanceUnit PROTOTYPE = DEFAULT;
+    /**
+     * Read a {@link DistanceUnit} from a {@link StreamInput}.
+     *
+     * @param in {@link StreamInput} to read the {@link DistanceUnit} from
+     * @return {@link DistanceUnit} read from the {@link StreamInput}
+     * @throws IOException if no unit can be read from the {@link StreamInput}
+     * @throws IllegalArgumentException if no matching {@link DistanceUnit} can be found
+     */
+    public static DistanceUnit readFromStream(StreamInput in) throws IOException {
+        byte b = in.readByte();
 
-    @Override
-    public DistanceUnit readFrom(StreamInput in) throws IOException {
-        int ordinal = in.readVInt();
-        if (ordinal < 0 || ordinal >= values().length) {
-            throw new IOException("Unknown DistanceUnit ordinal [" + ordinal + "]");
+        if (b < 0 || b >= values().length) {
+            throw new IllegalArgumentException("No type for distance unit matching [" + b + "]");
         }
-        return values()[ordinal];
-    }
-
-    public static DistanceUnit readUnitFrom(StreamInput in) throws IOException {
-        return PROTOTYPE.readFrom(in);
+        return values()[b];
     }
 
+    /**
+     * Write a {@link DistanceUnit} to a {@link StreamOutput}.
+     *
+     * @param out {@link StreamOutput} to write to
+     */
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(this.ordinal());
+        out.writeByte((byte) this.ordinal());
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/IndexFolderUpgrader.java b/core/src/main/java/org/elasticsearch/common/util/IndexFolderUpgrader.java
new file mode 100644
index 00000000000..221dc234511
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/util/IndexFolderUpgrader.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.util;
+
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+
+/**
+ * Renames index folders from {index.name} to {index.uuid}
+ */
+public class IndexFolderUpgrader {
+    private final NodeEnvironment nodeEnv;
+    private final Settings settings;
+    private final ESLogger logger = Loggers.getLogger(IndexFolderUpgrader.class);
+
+    /**
+     * Creates a new upgrader instance
+     * @param settings node settings
+     * @param nodeEnv the node env to operate on
+     */
+    IndexFolderUpgrader(Settings settings, NodeEnvironment nodeEnv) {
+        this.settings = settings;
+        this.nodeEnv = nodeEnv;
+    }
+
+    /**
+     * Moves the index folder found in <code>source</code> to <code>target</code>
+     */
+    void upgrade(final Index index, final Path source, final Path target) throws IOException {
+        boolean success = false;
+        try {
+            Files.move(source, target, StandardCopyOption.ATOMIC_MOVE);
+            success = true;
+        } catch (NoSuchFileException | FileNotFoundException exception) {
+            // thrown when the source is non-existent because the folder was renamed
+            // by another node (shared FS) after we checked if the target exists
+            logger.error("multiple nodes trying to upgrade [{}] in parallel, retry upgrading with single node",
+                exception, target);
+            throw exception;
+        } finally {
+            if (success) {
+                logger.info("{} moved from [{}] to [{}]", index, source, target);
+                logger.trace("{} syncing directory [{}]", index, target);
+                IOUtils.fsync(target, true);
+            }
+        }
+    }
+
+    /**
+     * Renames <code>indexFolderName</code> index folders found in node paths and custom path
+     * iff {@link #needsUpgrade(Index, String)} is true.
+     * Index folder in custom paths are renamed first followed by index folders in each node path.
+     */
+    void upgrade(final String indexFolderName) throws IOException {
+        for (NodeEnvironment.NodePath nodePath : nodeEnv.nodePaths()) {
+            final Path indexFolderPath = nodePath.indicesPath.resolve(indexFolderName);
+            final IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, indexFolderPath);
+            if (indexMetaData != null) {
+                final Index index = indexMetaData.getIndex();
+                if (needsUpgrade(index, indexFolderName)) {
+                    logger.info("{} upgrading [{}] to new naming convention", index, indexFolderPath);
+                    final IndexSettings indexSettings = new IndexSettings(indexMetaData, settings);
+                    if (indexSettings.hasCustomDataPath()) {
+                        // we rename index folder in custom path before renaming them in any node path
+                        // to have the index state under a not-yet-upgraded index folder, which we use to
+                        // continue renaming after a incomplete upgrade.
+                        final Path customLocationSource = nodeEnv.resolveBaseCustomLocation(indexSettings)
+                            .resolve(indexFolderName);
+                        final Path customLocationTarget = customLocationSource.resolveSibling(index.getUUID());
+                        // we rename the folder in custom path only the first time we encounter a state
+                        // in a node path, which needs upgrading, it is a no-op for subsequent node paths
+                        if (Files.exists(customLocationSource) // might not exist if no data was written for this index
+                            && Files.exists(customLocationTarget) == false) {
+                            upgrade(index, customLocationSource, customLocationTarget);
+                        } else {
+                            logger.info("[{}] no upgrade needed - already upgraded", customLocationTarget);
+                        }
+                    }
+                    upgrade(index, indexFolderPath, indexFolderPath.resolveSibling(index.getUUID()));
+                } else {
+                    logger.debug("[{}] no upgrade needed - already upgraded", indexFolderPath);
+                }
+            } else {
+                logger.warn("[{}] no index state found - ignoring", indexFolderPath);
+            }
+        }
+    }
+
+    /**
+     * Upgrades all indices found under <code>nodeEnv</code>. Already upgraded indices are ignored.
+     */
+    public static void upgradeIndicesIfNeeded(final Settings settings, final NodeEnvironment nodeEnv) throws IOException {
+        final IndexFolderUpgrader upgrader = new IndexFolderUpgrader(settings, nodeEnv);
+        for (String indexFolderName : nodeEnv.availableIndexFolders()) {
+            upgrader.upgrade(indexFolderName);
+        }
+    }
+
+    static boolean needsUpgrade(Index index, String indexFolderName) {
+        return indexFolderName.equals(index.getUUID()) == false;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
index 10b1412425c..df1288d4fd2 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.common.util.concurrent;
 
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 import java.util.Arrays;
@@ -41,7 +42,8 @@ public class EsExecutors {
      * Settings key to manually set the number of available processors.
      * This is used to adjust thread pools sizes etc. per node.
      */
-    public static final Setting<Integer> PROCESSORS_SETTING = Setting.intSetting("processors", Math.min(32, Runtime.getRuntime().availableProcessors()), 1, false, Setting.Scope.CLUSTER) ;
+    public static final Setting<Integer> PROCESSORS_SETTING =
+        Setting.intSetting("processors", Math.min(32, Runtime.getRuntime().availableProcessors()), 1, Property.NodeScope);
 
     /**
      * Returns the number of processors available but at most <tt>32</tt>.
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java
index fde8d828295..2f664679bb4 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java
@@ -40,11 +40,14 @@ public class EsThreadPoolExecutor extends ThreadPoolExecutor {
      */
     private final String name;
 
-    EsThreadPoolExecutor(String name, int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> workQueue, ThreadFactory threadFactory, ThreadContext contextHolder) {
+    EsThreadPoolExecutor(String name, int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit,
+            BlockingQueue<Runnable> workQueue, ThreadFactory threadFactory, ThreadContext contextHolder) {
         this(name, corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, new EsAbortPolicy(), contextHolder);
     }
 
-    EsThreadPoolExecutor(String name, int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> workQueue, ThreadFactory threadFactory, XRejectedExecutionHandler handler, ThreadContext contextHolder) {
+    EsThreadPoolExecutor(String name, int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit,
+            BlockingQueue<Runnable> workQueue, ThreadFactory threadFactory, XRejectedExecutionHandler handler,
+            ThreadContext contextHolder) {
         super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler);
         this.name = name;
         this.contextHolder = contextHolder;
@@ -133,112 +136,10 @@ public class EsThreadPoolExecutor extends ThreadPoolExecutor {
     }
 
     protected Runnable wrapRunnable(Runnable command) {
-        final Runnable wrappedCommand;
-        if (command instanceof AbstractRunnable) {
-            wrappedCommand = new FilterAbstractRunnable(contextHolder, (AbstractRunnable) command);
-        } else {
-            wrappedCommand = new FilterRunnable(contextHolder, command);
-        }
-        return wrappedCommand;
+        return contextHolder.preserveContext(command);
     }
 
     protected Runnable unwrap(Runnable runnable) {
-        if (runnable instanceof FilterAbstractRunnable) {
-            return ((FilterAbstractRunnable) runnable).in;
-        } else if (runnable instanceof FilterRunnable) {
-            return ((FilterRunnable) runnable).in;
-        }
-        return runnable;
+        return contextHolder.unwrap(runnable);
     }
-
-    private  class FilterAbstractRunnable extends AbstractRunnable {
-        private final ThreadContext contextHolder;
-        private final AbstractRunnable in;
-        private final ThreadContext.StoredContext ctx;
-
-        FilterAbstractRunnable(ThreadContext contextHolder, AbstractRunnable in) {
-            this.contextHolder = contextHolder;
-            ctx = contextHolder.newStoredContext();
-            this.in = in;
-        }
-
-        @Override
-        public boolean isForceExecution() {
-            return in.isForceExecution();
-        }
-
-        @Override
-        public void onAfter() {
-            in.onAfter();
-        }
-
-        @Override
-        public void onFailure(Throwable t) {
-            in.onFailure(t);
-        }
-
-        @Override
-        public void onRejection(Throwable t) {
-            in.onRejection(t);
-        }
-
-        @Override
-        protected void doRun() throws Exception {
-            boolean whileRunning = false;
-            try (ThreadContext.StoredContext ingore = contextHolder.stashContext()){
-                ctx.restore();
-                whileRunning = true;
-                in.doRun();
-                whileRunning = false;
-            } catch (IllegalStateException ex) {
-                if (whileRunning || isShutdown() == false) {
-                    throw ex;
-                }
-                // if we hit an ISE here we have been shutting down
-                // this comes from the threadcontext and barfs if
-                // our threadpool has been shutting down
-            }
-        }
-
-        @Override
-        public String toString() {
-            return in.toString();
-        }
-
-    }
-
-    private  class FilterRunnable implements Runnable {
-        private final ThreadContext contextHolder;
-        private final Runnable in;
-        private final ThreadContext.StoredContext ctx;
-
-        FilterRunnable(ThreadContext contextHolder, Runnable in) {
-            this.contextHolder = contextHolder;
-            ctx = contextHolder.newStoredContext();
-            this.in = in;
-        }
-
-        @Override
-        public void run() {
-            boolean whileRunning = false;
-            try (ThreadContext.StoredContext ingore = contextHolder.stashContext()){
-                ctx.restore();
-                whileRunning = true;
-                in.run();
-                whileRunning = false;
-            } catch (IllegalStateException ex) {
-                if (whileRunning || isShutdown() == false) {
-                    throw ex;
-                }
-                // if we hit an ISE here we have been shutting down
-                // this comes from the threadcontext and barfs if
-                // our threadpool has been shutting down
-            }
-        }
-        @Override
-        public String toString() {
-            return in.toString();
-        }
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/KeyedLock.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/KeyedLock.java
index 83bb9fd690d..5c30330c156 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/KeyedLock.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/KeyedLock.java
@@ -20,7 +20,10 @@
 package org.elasticsearch.common.util.concurrent;
 
 
+import org.elasticsearch.common.lease.Releasable;
+
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.ReentrantLock;
 
@@ -29,9 +32,8 @@ import java.util.concurrent.locks.ReentrantLock;
  * created the first time they are acquired and removed if no thread hold the
  * lock. The latter is important to assure that the list of locks does not grow
  * infinitely.
- * 
- * A Thread can acquire a lock only once.
- * 
+ *
+ *
  * */
 public class KeyedLock<T> {
 
@@ -50,48 +52,38 @@ public class KeyedLock<T> {
 
     private final ConcurrentMap<T, KeyLock> map = ConcurrentCollections.newConcurrentMap();
 
-    protected final ThreadLocal<KeyLock> threadLocal = new ThreadLocal<>();
-
-    public void acquire(T key) {
+    public Releasable acquire(T key) {
+        assert isHeldByCurrentThread(key) == false : "lock for " + key + " is already heald by this thread";
         while (true) {
-            if (threadLocal.get() != null) {
-                // if we are here, the thread already has the lock
-                throw new IllegalStateException("Lock already acquired in Thread" + Thread.currentThread().getId()
-                        + " for key " + key);
-            }
             KeyLock perNodeLock = map.get(key);
             if (perNodeLock == null) {
                 KeyLock newLock = new KeyLock(fair);
                 perNodeLock = map.putIfAbsent(key, newLock);
                 if (perNodeLock == null) {
                     newLock.lock();
-                    threadLocal.set(newLock);
-                    return;
+                    return new ReleasableLock(key, newLock);
                 }
             }
             assert perNodeLock != null;
             int i = perNodeLock.count.get();
             if (i > 0 && perNodeLock.count.compareAndSet(i, i + 1)) {
                 perNodeLock.lock();
-                threadLocal.set(perNodeLock);
-                return;
+                return new ReleasableLock(key, perNodeLock);
             }
         }
     }
 
-    public void release(T key) {
-        KeyLock lock = threadLocal.get();
+    public boolean isHeldByCurrentThread(T key) {
+        KeyLock lock = map.get(key);
         if (lock == null) {
-            throw new IllegalStateException("Lock not acquired");
+            return false;
         }
-        release(key, lock);
+        return lock.isHeldByCurrentThread();
     }
 
     void release(T key, KeyLock lock) {
-        assert lock.isHeldByCurrentThread();
         assert lock == map.get(key);
         lock.unlock();
-        threadLocal.set(null);
         int decrementAndGet = lock.count.decrementAndGet();
         if (decrementAndGet == 0) {
             map.remove(key, lock);
@@ -99,6 +91,24 @@ public class KeyedLock<T> {
     }
 
 
+    private final class ReleasableLock implements Releasable {
+        final T key;
+        final KeyLock lock;
+        final AtomicBoolean closed = new AtomicBoolean();
+
+        private ReleasableLock(T key, KeyLock lock) {
+            this.key = key;
+            this.lock = lock;
+        }
+
+        @Override
+        public void close() {
+            if (closed.compareAndSet(false, true)) {
+                release(key, lock);
+            }
+        }
+    }
+
     @SuppressWarnings("serial")
     private final static class KeyLock extends ReentrantLock {
         KeyLock(boolean fair) {
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
index 1928392fe41..462b4f539dc 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
@@ -19,11 +19,11 @@
 package org.elasticsearch.common.util.concurrent;
 
 import org.apache.lucene.util.CloseableThreadLocal;
-import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 import java.io.Closeable;
@@ -63,7 +63,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 public final class ThreadContext implements Closeable, Writeable<ThreadContext.ThreadContextStruct>{
 
     public static final String PREFIX = "request.headers";
-    public static final Setting<Settings> DEFAULT_HEADERS_SETTING = Setting.groupSetting(PREFIX + ".", false, Setting.Scope.CLUSTER);
+    public static final Setting<Settings> DEFAULT_HEADERS_SETTING = Setting.groupSetting(PREFIX + ".", Property.NodeScope);
     private final Map<String, String> defaultHeader;
     private static final ThreadContextStruct DEFAULT_CONTEXT = new ThreadContextStruct(Collections.emptyMap());
     private final ContextThreadLocal threadLocal;
@@ -200,6 +200,36 @@ public final class ThreadContext implements Closeable, Writeable<ThreadContext.T
         return (T) threadLocal.get().transientHeaders.get(key);
     }
 
+    /**
+     * Saves the current thread context and wraps command in a Runnable that restores that context before running command. If
+     * <code>command</code> has already been passed through this method then it is returned unaltered rather than wrapped twice.
+     */
+    public Runnable preserveContext(Runnable command) {
+        if (command instanceof ContextPreservingAbstractRunnable) {
+            return command;
+        }
+        if (command instanceof ContextPreservingRunnable) {
+            return command;
+        }
+        if (command instanceof AbstractRunnable) {
+            return new ContextPreservingAbstractRunnable((AbstractRunnable) command);
+        }
+        return new ContextPreservingRunnable(command);
+    }
+
+    /**
+     * Unwraps a command that was previously wrapped by {@link #preserveContext(Runnable)}.
+     */
+    public Runnable unwrap(Runnable command) {
+        if (command instanceof ContextPreservingAbstractRunnable) {
+            return ((ContextPreservingAbstractRunnable) command).unwrap();
+        }
+        if (command instanceof ContextPreservingRunnable) {
+            return ((ContextPreservingRunnable) command).unwrap();
+        }
+        return command;
+    }
+
     public interface StoredContext extends AutoCloseable {
         @Override
         void close();
@@ -356,4 +386,104 @@ public final class ThreadContext implements Closeable, Writeable<ThreadContext.T
             }
         }
     }
+
+    /**
+     * Wraps a Runnable to preserve the thread context.
+     */
+    class ContextPreservingRunnable implements Runnable {
+        private final Runnable in;
+        private final ThreadContext.StoredContext ctx;
+
+        ContextPreservingRunnable(Runnable in) {
+            ctx = newStoredContext();
+            this.in = in;
+        }
+
+        @Override
+        public void run() {
+            boolean whileRunning = false;
+            try (ThreadContext.StoredContext ingore = stashContext()){
+                ctx.restore();
+                whileRunning = true;
+                in.run();
+                whileRunning = false;
+            } catch (IllegalStateException ex) {
+                if (whileRunning || threadLocal.closed.get() == false) {
+                    throw ex;
+                }
+                // if we hit an ISE here we have been shutting down
+                // this comes from the threadcontext and barfs if
+                // our threadpool has been shutting down
+            }
+        }
+
+        @Override
+        public String toString() {
+            return in.toString();
+        }
+
+        public Runnable unwrap() {
+            return in;
+        }
+    }
+
+    /**
+     * Wraps an AbstractRunnable to preserve the thread context.
+     */
+    public class ContextPreservingAbstractRunnable extends AbstractRunnable {
+        private final AbstractRunnable in;
+        private final ThreadContext.StoredContext ctx;
+
+        private ContextPreservingAbstractRunnable(AbstractRunnable in) {
+            ctx = newStoredContext();
+            this.in = in;
+        }
+
+        @Override
+        public boolean isForceExecution() {
+            return in.isForceExecution();
+        }
+
+        @Override
+        public void onAfter() {
+            in.onAfter();
+        }
+
+        @Override
+        public void onFailure(Throwable t) {
+            in.onFailure(t);
+        }
+
+        @Override
+        public void onRejection(Throwable t) {
+            in.onRejection(t);
+        }
+
+        @Override
+        protected void doRun() throws Exception {
+            boolean whileRunning = false;
+            try (ThreadContext.StoredContext ingore = stashContext()){
+                ctx.restore();
+                whileRunning = true;
+                in.doRun();
+                whileRunning = false;
+            } catch (IllegalStateException ex) {
+                if (whileRunning || threadLocal.closed.get() == false) {
+                    throw ex;
+                }
+                // if we hit an ISE here we have been shutting down
+                // this comes from the threadcontext and barfs if
+                // our threadpool has been shutting down
+            }
+        }
+
+        @Override
+        public String toString() {
+            return in.toString();
+        }
+
+        public AbstractRunnable unwrap() {
+            return in;
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
index 8ca53af186c..2229d45840b 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
@@ -961,6 +961,10 @@ public final class XContentBuilder implements BytesStream, Releasable {
         return this;
     }
 
+    public XContentBuilder timeValueField(String rawFieldName, String readableFieldName, TimeValue timeValue) throws IOException {
+        return timeValueField(rawFieldName, readableFieldName, timeValue.millis(), TimeUnit.MILLISECONDS);
+    }
+
     public XContentBuilder timeValueField(String rawFieldName, String readableFieldName, long rawTime, TimeUnit timeUnit) throws
         IOException {
         if (humanReadable) {
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
index fd4aa782302..e994b81832f 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.common.xcontent.support;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -286,14 +287,21 @@ public abstract class AbstractXContentParser implements XContentParser {
 
     static List<Object> readList(XContentParser parser, MapFactory mapFactory) throws IOException {
         XContentParser.Token token = parser.currentToken();
+        if (token == null) {
+            token = parser.nextToken();
+        }
         if (token == XContentParser.Token.FIELD_NAME) {
             token = parser.nextToken();
         }
         if (token == XContentParser.Token.START_ARRAY) {
             token = parser.nextToken();
+        } else {
+            throw new ElasticsearchParseException("Failed to parse list:  expecting "
+                    + XContentParser.Token.START_ARRAY + " but got " + token);
         }
+
         ArrayList<Object> list = new ArrayList<>();
-        for (; token != XContentParser.Token.END_ARRAY; token = parser.nextToken()) {
+        for (; token != null && token != XContentParser.Token.END_ARRAY; token = parser.nextToken()) {
             list.add(readValue(parser, mapFactory, token));
         }
         return list;
diff --git a/core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java b/core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
index e08757a3f2a..4076b880d6f 100644
--- a/core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
+++ b/core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
@@ -23,6 +23,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.ExtensionPoint;
 import org.elasticsearch.discovery.local.LocalDiscovery;
@@ -45,10 +46,11 @@ import java.util.function.Function;
  */
 public class DiscoveryModule extends AbstractModule {
 
-    public static final Setting<String> DISCOVERY_TYPE_SETTING = new Setting<>("discovery.type",
-        settings -> DiscoveryNode.localNode(settings) ? "local" : "zen", Function.identity(), false, Setting.Scope.CLUSTER);
-    public static final Setting<String> ZEN_MASTER_SERVICE_TYPE_SETTING = new Setting<>("discovery.zen.masterservice.type",
-            "zen", Function.identity(), false, Setting.Scope.CLUSTER);
+    public static final Setting<String> DISCOVERY_TYPE_SETTING =
+        new Setting<>("discovery.type", settings -> DiscoveryNode.localNode(settings) ? "local" : "zen", Function.identity(),
+            Property.NodeScope);
+    public static final Setting<String> ZEN_MASTER_SERVICE_TYPE_SETTING =
+        new Setting<>("discovery.zen.masterservice.type", "zen", Function.identity(), Property.NodeScope);
 
     private final Settings settings;
     private final Map<String, List<Class<? extends UnicastHostsProvider>>> unicastHostProviders = new HashMap<>();
diff --git a/core/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java b/core/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java
index b899f0a8afc..ca7ab342cd5 100644
--- a/core/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java
+++ b/core/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java
@@ -24,6 +24,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.RestStatus;
@@ -42,16 +43,25 @@ public class DiscoverySettings extends AbstractComponent {
      * sets the timeout for a complete publishing cycle, including both sending and committing. the master
      * will continue to process the next cluster state update after this time has elapsed
      **/
-    public static final Setting<TimeValue> PUBLISH_TIMEOUT_SETTING = Setting.positiveTimeSetting("discovery.zen.publish_timeout", TimeValue.timeValueSeconds(30), true, Setting.Scope.CLUSTER);
+    public static final Setting<TimeValue> PUBLISH_TIMEOUT_SETTING =
+        Setting.positiveTimeSetting("discovery.zen.publish_timeout", TimeValue.timeValueSeconds(30),
+            Property.Dynamic, Property.NodeScope);
 
     /**
      * sets the timeout for receiving enough acks for a specific cluster state and committing it. failing
      * to receive responses within this window will cause the cluster state change to be rejected.
      */
-    public static final Setting<TimeValue> COMMIT_TIMEOUT_SETTING = new Setting<>("discovery.zen.commit_timeout", (s) -> PUBLISH_TIMEOUT_SETTING.getRaw(s), (s) -> TimeValue.parseTimeValue(s, TimeValue.timeValueSeconds(30), "discovery.zen.commit_timeout"), true, Setting.Scope.CLUSTER);
-    public static final Setting<ClusterBlock> NO_MASTER_BLOCK_SETTING = new Setting<>("discovery.zen.no_master_block", "write", DiscoverySettings::parseNoMasterBlock, true, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> PUBLISH_DIFF_ENABLE_SETTING = Setting.boolSetting("discovery.zen.publish_diff.enable", true, true, Setting.Scope.CLUSTER);
-    public static final Setting<TimeValue> INITIAL_STATE_TIMEOUT_SETTING = Setting.positiveTimeSetting("discovery.initial_state_timeout", TimeValue.timeValueSeconds(30), false, Setting.Scope.CLUSTER);
+    public static final Setting<TimeValue> COMMIT_TIMEOUT_SETTING =
+        new Setting<>("discovery.zen.commit_timeout", (s) -> PUBLISH_TIMEOUT_SETTING.getRaw(s),
+            (s) -> TimeValue.parseTimeValue(s, TimeValue.timeValueSeconds(30), "discovery.zen.commit_timeout"),
+            Property.Dynamic, Property.NodeScope);
+    public static final Setting<ClusterBlock> NO_MASTER_BLOCK_SETTING =
+        new Setting<>("discovery.zen.no_master_block", "write", DiscoverySettings::parseNoMasterBlock,
+            Property.Dynamic, Property.NodeScope);
+    public static final Setting<Boolean> PUBLISH_DIFF_ENABLE_SETTING =
+        Setting.boolSetting("discovery.zen.publish_diff.enable", true, Property.Dynamic, Property.NodeScope);
+    public static final Setting<TimeValue> INITIAL_STATE_TIMEOUT_SETTING =
+        Setting.positiveTimeSetting("discovery.initial_state_timeout", TimeValue.timeValueSeconds(30), Property.NodeScope);
 
     private volatile ClusterBlock noMasterBlock;
     private volatile TimeValue publishTimeout;
diff --git a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
index 0462d6a8d8d..cf697871d35 100644
--- a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
@@ -21,7 +21,6 @@ package org.elasticsearch.discovery.local;
 
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.Diff;
@@ -31,6 +30,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
index 19a2cf06bf4..0edbf8841ad 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.discovery.zen;
 
 import org.elasticsearch.ElasticsearchTimeoutException;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.NotMasterException;
@@ -28,7 +27,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.cluster.service.InternalClusterService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
@@ -346,7 +345,7 @@ public class NodeJoinController extends AbstractComponent {
         }
 
         private void assertClusterStateThread() {
-            assert clusterService instanceof InternalClusterService == false || ((InternalClusterService) clusterService).assertClusterStateThread();
+            assert clusterService instanceof ClusterService == false || ((ClusterService) clusterService).assertClusterStateThread();
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index c0dd78b4e5f..6e0f17812ce 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -24,7 +24,6 @@ import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.NotMasterException;
@@ -35,7 +34,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.cluster.service.InternalClusterService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.component.Lifecycle;
@@ -46,6 +45,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.Discovery;
@@ -71,6 +71,7 @@ import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -78,6 +79,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 
@@ -86,17 +88,26 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
  */
 public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implements Discovery, PingContextProvider {
 
-    public final static Setting<TimeValue> PING_TIMEOUT_SETTING = Setting.positiveTimeSetting("discovery.zen.ping_timeout", timeValueSeconds(3), false, Setting.Scope.CLUSTER);
-    public final static Setting<TimeValue> JOIN_TIMEOUT_SETTING = Setting.timeSetting("discovery.zen.join_timeout",
-            settings -> TimeValue.timeValueMillis(PING_TIMEOUT_SETTING.get(settings).millis() * 20).toString(), TimeValue.timeValueMillis(0), false, Setting.Scope.CLUSTER);
-    public final static Setting<Integer> JOIN_RETRY_ATTEMPTS_SETTING = Setting.intSetting("discovery.zen.join_retry_attempts", 3, 1, false, Setting.Scope.CLUSTER);
-    public final static Setting<TimeValue> JOIN_RETRY_DELAY_SETTING = Setting.positiveTimeSetting("discovery.zen.join_retry_delay", TimeValue.timeValueMillis(100), false, Setting.Scope.CLUSTER);
-    public final static Setting<Integer> MAX_PINGS_FROM_ANOTHER_MASTER_SETTING = Setting.intSetting("discovery.zen.max_pings_from_another_master", 3, 1, false, Setting.Scope.CLUSTER);
-    public final static Setting<Boolean> SEND_LEAVE_REQUEST_SETTING = Setting.boolSetting("discovery.zen.send_leave_request", true, false, Setting.Scope.CLUSTER);
-    public final static Setting<Boolean> MASTER_ELECTION_FILTER_CLIENT_SETTING = Setting.boolSetting("discovery.zen.master_election.filter_client", true, false, Setting.Scope.CLUSTER);
-    public final static Setting<TimeValue> MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING = Setting.timeSetting("discovery.zen.master_election.wait_for_joins_timeout",
-            settings -> TimeValue.timeValueMillis(JOIN_TIMEOUT_SETTING.get(settings).millis() / 2).toString(), TimeValue.timeValueMillis(0), false, Setting.Scope.CLUSTER);
-    public final static Setting<Boolean> MASTER_ELECTION_FILTER_DATA_SETTING = Setting.boolSetting("discovery.zen.master_election.filter_data", false, false, Setting.Scope.CLUSTER);
+    public final static Setting<TimeValue> PING_TIMEOUT_SETTING =
+        Setting.positiveTimeSetting("discovery.zen.ping_timeout", timeValueSeconds(3), Property.NodeScope);
+    public final static Setting<TimeValue> JOIN_TIMEOUT_SETTING =
+        Setting.timeSetting("discovery.zen.join_timeout",
+            settings -> TimeValue.timeValueMillis(PING_TIMEOUT_SETTING.get(settings).millis() * 20).toString(),
+            TimeValue.timeValueMillis(0), Property.NodeScope);
+    public final static Setting<Integer> JOIN_RETRY_ATTEMPTS_SETTING =
+        Setting.intSetting("discovery.zen.join_retry_attempts", 3, 1, Property.NodeScope);
+    public final static Setting<TimeValue> JOIN_RETRY_DELAY_SETTING =
+        Setting.positiveTimeSetting("discovery.zen.join_retry_delay", TimeValue.timeValueMillis(100), Property.NodeScope);
+    public final static Setting<Integer> MAX_PINGS_FROM_ANOTHER_MASTER_SETTING =
+        Setting.intSetting("discovery.zen.max_pings_from_another_master", 3, 1, Property.NodeScope);
+    public final static Setting<Boolean> SEND_LEAVE_REQUEST_SETTING =
+        Setting.boolSetting("discovery.zen.send_leave_request", true, Property.NodeScope);
+    public final static Setting<TimeValue> MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING =
+        Setting.timeSetting("discovery.zen.master_election.wait_for_joins_timeout",
+            settings -> TimeValue.timeValueMillis(JOIN_TIMEOUT_SETTING.get(settings).millis() / 2).toString(), TimeValue.timeValueMillis(0),
+            Property.NodeScope);
+    public final static Setting<Boolean> MASTER_ELECTION_IGNORE_NON_MASTER_PINGS_SETTING =
+            Setting.boolSetting("discovery.zen.master_election.ignore_non_master_pings", false, Property.NodeScope);
 
     public static final String DISCOVERY_REJOIN_ACTION_NAME = "internal:discovery/zen/rejoin";
 
@@ -127,8 +138,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
 
     private final ElectMasterService electMaster;
 
-    private final boolean masterElectionFilterClientNodes;
-    private final boolean masterElectionFilterDataNodes;
+    private final boolean masterElectionIgnoreNonMasters;
     private final TimeValue masterElectionWaitForJoinsTimeout;
 
     private final JoinThreadControl joinThreadControl;
@@ -158,11 +168,11 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
         this.maxPingsFromAnotherMaster = MAX_PINGS_FROM_ANOTHER_MASTER_SETTING.get(settings);
         this.sendLeaveRequest = SEND_LEAVE_REQUEST_SETTING.get(settings);
 
-        this.masterElectionFilterClientNodes = MASTER_ELECTION_FILTER_CLIENT_SETTING.get(settings);
-        this.masterElectionFilterDataNodes = MASTER_ELECTION_FILTER_DATA_SETTING.get(settings);
+        this.masterElectionIgnoreNonMasters = MASTER_ELECTION_IGNORE_NON_MASTER_PINGS_SETTING.get(settings);
         this.masterElectionWaitForJoinsTimeout = MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING.get(settings);
 
-        logger.debug("using ping_timeout [{}], join.timeout [{}], master_election.filter_client [{}], master_election.filter_data [{}]", this.pingTimeout, joinTimeout, masterElectionFilterClientNodes, masterElectionFilterDataNodes);
+        logger.debug("using ping_timeout [{}], join.timeout [{}], master_election.ignore_non_master [{}]",
+                this.pingTimeout, joinTimeout, masterElectionIgnoreNonMasters);
 
         clusterSettings.addSettingsUpdateConsumer(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING, this::handleMinimumMasterNodesChanged, (value) -> {
             final ClusterState clusterState = clusterService.state();
@@ -823,7 +833,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
             return null;
         }
         if (logger.isTraceEnabled()) {
-            StringBuilder sb = new StringBuilder("full ping responses:");
+            StringBuilder sb = new StringBuilder();
             if (fullPingResponses.length == 0) {
                 sb.append(" {none}");
             } else {
@@ -831,33 +841,12 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
                     sb.append("\n\t--> ").append(pingResponse);
                 }
             }
-            logger.trace(sb.toString());
+            logger.trace("full ping responses:{}", sb);
         }
 
         // filter responses
-        List<ZenPing.PingResponse> pingResponses = new ArrayList<>();
-        for (ZenPing.PingResponse pingResponse : fullPingResponses) {
-            DiscoveryNode node = pingResponse.node();
-            if (masterElectionFilterClientNodes && (node.clientNode() || (!node.masterNode() && !node.dataNode()))) {
-                // filter out the client node, which is a client node, or also one that is not data and not master (effectively, client)
-            } else if (masterElectionFilterDataNodes && (!node.masterNode() && node.dataNode())) {
-                // filter out data node that is not also master
-            } else {
-                pingResponses.add(pingResponse);
-            }
-        }
-
-        if (logger.isDebugEnabled()) {
-            StringBuilder sb = new StringBuilder("filtered ping responses: (filter_client[").append(masterElectionFilterClientNodes).append("], filter_data[").append(masterElectionFilterDataNodes).append("])");
-            if (pingResponses.isEmpty()) {
-                sb.append(" {none}");
-            } else {
-                for (ZenPing.PingResponse pingResponse : pingResponses) {
-                    sb.append("\n\t--> ").append(pingResponse);
-                }
-            }
-            logger.debug(sb.toString());
-        }
+        final List<ZenPing.PingResponse> pingResponses;
+        pingResponses = filterPingResponses(fullPingResponses, masterElectionIgnoreNonMasters, logger);
 
         final DiscoveryNode localNode = clusterService.localNode();
         List<DiscoveryNode> pingMasters = new ArrayList<>();
@@ -913,12 +902,34 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
         }
     }
 
+    static List<ZenPing.PingResponse> filterPingResponses(ZenPing.PingResponse[] fullPingResponses, boolean masterElectionIgnoreNonMasters, ESLogger logger) {
+        List<ZenPing.PingResponse> pingResponses;
+        if (masterElectionIgnoreNonMasters) {
+            pingResponses = Arrays.stream(fullPingResponses).filter(ping -> ping.node().isMasterNode()).collect(Collectors.toList());
+        } else {
+            pingResponses = Arrays.asList(fullPingResponses);
+        }
+
+        if (logger.isDebugEnabled()) {
+            StringBuilder sb = new StringBuilder();
+            if (pingResponses.isEmpty()) {
+                sb.append(" {none}");
+            } else {
+                for (ZenPing.PingResponse pingResponse : pingResponses) {
+                    sb.append("\n\t--> ").append(pingResponse);
+                }
+            }
+            logger.debug("filtered ping responses: (ignore_non_masters [{}]){}", masterElectionIgnoreNonMasters, sb);
+        }
+        return pingResponses;
+    }
+
     protected ClusterState rejoin(ClusterState clusterState, String reason) {
 
         // *** called from within an cluster state update task *** //
-        assert Thread.currentThread().getName().contains(InternalClusterService.UPDATE_THREAD_NAME);
+        assert Thread.currentThread().getName().contains(ClusterService.UPDATE_THREAD_NAME);
 
-        logger.warn(reason + ", current nodes: {}", clusterState.nodes());
+        logger.warn("{}, current nodes: {}", reason, clusterState.nodes());
         nodesFD.stop();
         masterFD.stop(reason);
 
@@ -946,7 +957,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
 
     private ClusterState handleAnotherMaster(ClusterState localClusterState, final DiscoveryNode otherMaster, long otherClusterStateVersion, String reason) {
         assert localClusterState.nodes().localNodeMaster() : "handleAnotherMaster called but current node is not a master";
-        assert Thread.currentThread().getName().contains(InternalClusterService.UPDATE_THREAD_NAME) : "not called from the cluster state update thread";
+        assert Thread.currentThread().getName().contains(ClusterService.UPDATE_THREAD_NAME) : "not called from the cluster state update thread";
 
         if (otherClusterStateVersion > localClusterState.version()) {
             return rejoin(localClusterState, "zen-disco-discovered another master with a new cluster_state [" + otherMaster + "][" + reason + "]");
@@ -1184,7 +1195,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
         }
 
         private void assertClusterStateThread() {
-            assert clusterService instanceof InternalClusterService == false || ((InternalClusterService) clusterService).assertClusterStateThread();
+            assert clusterService instanceof ClusterService == false || ((ClusterService) clusterService).assertClusterStateThread();
         }
 
     }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java b/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java
index 1482fb92a22..a3da8be5a94 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java
@@ -26,6 +26,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 
@@ -40,7 +41,8 @@ import java.util.List;
  */
 public class ElectMasterService extends AbstractComponent {
 
-    public static final Setting<Integer> DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING = Setting.intSetting("discovery.zen.minimum_master_nodes", -1, true, Setting.Scope.CLUSTER);
+    public static final Setting<Integer> DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING =
+        Setting.intSetting("discovery.zen.minimum_master_nodes", -1, Property.Dynamic, Property.NodeScope);
 
     // This is the minimum version a master needs to be on, otherwise it gets ignored
     // This is based on the minimum compatible version of the current version this node is on
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/FaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/FaultDetection.java
index 62b0250315c..1cfd46634a5 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/FaultDetection.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/FaultDetection.java
@@ -22,7 +22,7 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Scope;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -37,11 +37,16 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
  */
 public abstract class FaultDetection extends AbstractComponent {
 
-    public static final Setting<Boolean> CONNECT_ON_NETWORK_DISCONNECT_SETTING = Setting.boolSetting("discovery.zen.fd.connect_on_network_disconnect", false, false, Scope.CLUSTER);
-    public static final Setting<TimeValue> PING_INTERVAL_SETTING = Setting.positiveTimeSetting("discovery.zen.fd.ping_interval", timeValueSeconds(1), false, Scope.CLUSTER);
-    public static final Setting<TimeValue> PING_TIMEOUT_SETTING = Setting.timeSetting("discovery.zen.fd.ping_timeout", timeValueSeconds(30), false, Scope.CLUSTER);
-    public static final Setting<Integer> PING_RETRIES_SETTING = Setting.intSetting("discovery.zen.fd.ping_retries", 3, false, Scope.CLUSTER);
-    public static final Setting<Boolean> REGISTER_CONNECTION_LISTENER_SETTING = Setting.boolSetting("discovery.zen.fd.register_connection_listener", true, false, Scope.CLUSTER);
+    public static final Setting<Boolean> CONNECT_ON_NETWORK_DISCONNECT_SETTING =
+        Setting.boolSetting("discovery.zen.fd.connect_on_network_disconnect", false, Property.NodeScope);
+    public static final Setting<TimeValue> PING_INTERVAL_SETTING =
+        Setting.positiveTimeSetting("discovery.zen.fd.ping_interval", timeValueSeconds(1), Property.NodeScope);
+    public static final Setting<TimeValue> PING_TIMEOUT_SETTING =
+        Setting.timeSetting("discovery.zen.fd.ping_timeout", timeValueSeconds(30), Property.NodeScope);
+    public static final Setting<Integer> PING_RETRIES_SETTING =
+        Setting.intSetting("discovery.zen.fd.ping_retries", 3, Property.NodeScope);
+    public static final Setting<Boolean> REGISTER_CONNECTION_LISTENER_SETTING =
+        Setting.boolSetting("discovery.zen.fd.register_connection_listener", true, Property.NodeScope);
 
     protected final ThreadPool threadPool;
     protected final ClusterName clusterName;
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
index 73be1d3bb28..96ed7f76419 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
@@ -21,12 +21,12 @@ package org.elasticsearch.discovery.zen.fd;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.NotMasterException;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java b/core/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java
index 04af8207c37..de4caf664ea 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.discovery.zen.membership;
 
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
index 427abca8d85..0e9b81ad1fc 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
@@ -32,6 +32,7 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
@@ -86,8 +87,11 @@ import static org.elasticsearch.discovery.zen.ping.ZenPing.PingResponse.readPing
 public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implements ZenPing {
 
     public static final String ACTION_NAME = "internal:discovery/zen/unicast";
-    public static final Setting<List<String>> DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING = Setting.listSetting("discovery.zen.ping.unicast.hosts", Collections.emptyList(), Function.identity(), false, Setting.Scope.CLUSTER);
-    public static final Setting<Integer> DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING = Setting.intSetting("discovery.zen.ping.unicast.concurrent_connects", 10, 0, false, Setting.Scope.CLUSTER);
+    public static final Setting<List<String>> DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING =
+        Setting.listSetting("discovery.zen.ping.unicast.hosts", Collections.emptyList(), Function.identity(),
+            Property.NodeScope);
+    public static final Setting<Integer> DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING =
+        Setting.intSetting("discovery.zen.ping.unicast.concurrent_connects", 10, 0, Property.NodeScope);
 
     // these limits are per-address
     public static final int LIMIT_FOREIGN_PORTS_COUNT = 1;
diff --git a/core/src/main/java/org/elasticsearch/env/Environment.java b/core/src/main/java/org/elasticsearch/env/Environment.java
index 1f8cffc97f3..e022ce6ad2f 100644
--- a/core/src/main/java/org/elasticsearch/env/Environment.java
+++ b/core/src/main/java/org/elasticsearch/env/Environment.java
@@ -23,6 +23,7 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 import java.io.IOException;
@@ -46,15 +47,17 @@ import static org.elasticsearch.common.Strings.cleanPath;
 // TODO: move PathUtils to be package-private here instead of
 // public+forbidden api!
 public class Environment {
-    public static final Setting<String> PATH_HOME_SETTING = Setting.simpleString("path.home", false, Setting.Scope.CLUSTER);
-    public static final Setting<String> PATH_CONF_SETTING = Setting.simpleString("path.conf", false, Setting.Scope.CLUSTER);
-    public static final Setting<String> PATH_SCRIPTS_SETTING = Setting.simpleString("path.scripts", false, Setting.Scope.CLUSTER);
-    public static final Setting<List<String>> PATH_DATA_SETTING = Setting.listSetting("path.data", Collections.emptyList(), Function.identity(), false, Setting.Scope.CLUSTER);
-    public static final Setting<String> PATH_LOGS_SETTING = Setting.simpleString("path.logs", false, Setting.Scope.CLUSTER);
-    public static final Setting<String> PATH_PLUGINS_SETTING = Setting.simpleString("path.plugins", false, Setting.Scope.CLUSTER);
-    public static final Setting<List<String>> PATH_REPO_SETTING = Setting.listSetting("path.repo", Collections.emptyList(), Function.identity(), false, Setting.Scope.CLUSTER);
-    public static final Setting<String> PATH_SHARED_DATA_SETTING = Setting.simpleString("path.shared_data", false, Setting.Scope.CLUSTER);
-    public static final Setting<String> PIDFILE_SETTING = Setting.simpleString("pidfile", false, Setting.Scope.CLUSTER);
+    public static final Setting<String> PATH_HOME_SETTING = Setting.simpleString("path.home", Property.NodeScope);
+    public static final Setting<String> PATH_CONF_SETTING = Setting.simpleString("path.conf", Property.NodeScope);
+    public static final Setting<String> PATH_SCRIPTS_SETTING = Setting.simpleString("path.scripts", Property.NodeScope);
+    public static final Setting<List<String>> PATH_DATA_SETTING =
+        Setting.listSetting("path.data", Collections.emptyList(), Function.identity(), Property.NodeScope);
+    public static final Setting<String> PATH_LOGS_SETTING = Setting.simpleString("path.logs", Property.NodeScope);
+    public static final Setting<String> PATH_PLUGINS_SETTING = Setting.simpleString("path.plugins", Property.NodeScope);
+    public static final Setting<List<String>> PATH_REPO_SETTING =
+        Setting.listSetting("path.repo", Collections.emptyList(), Function.identity(), Property.NodeScope);
+    public static final Setting<String> PATH_SHARED_DATA_SETTING = Setting.simpleString("path.shared_data", Property.NodeScope);
+    public static final Setting<String> PIDFILE_SETTING = Setting.simpleString("pidfile", Property.NodeScope);
 
     private final Settings settings;
 
diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
index 0eec5c5765e..c6eec09b1c8 100644
--- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
+++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
@@ -36,7 +36,7 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Scope;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
@@ -49,7 +49,6 @@ import org.elasticsearch.index.store.FsDirectoryService;
 import org.elasticsearch.monitor.fs.FsInfo;
 import org.elasticsearch.monitor.fs.FsProbe;
 import org.elasticsearch.monitor.jvm.JvmInfo;
-import org.elasticsearch.monitor.process.ProcessProbe;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -71,7 +70,6 @@ import java.util.Set;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.stream.Collectors;
 
 import static java.util.Collections.unmodifiableSet;
 
@@ -90,7 +88,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
          *  not running on Linux, or we hit an exception trying), True means the device possibly spins and False means it does not. */
         public final Boolean spins;
 
-        public NodePath(Path path, Environment environment) throws IOException {
+        public NodePath(Path path) throws IOException {
             this.path = path;
             this.indicesPath = path.resolve(INDICES_FOLDER);
             this.fileStore = Environment.getFileStore(path);
@@ -103,16 +101,18 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
 
         /**
          * Resolves the given shards directory against this NodePath
+         * ${data.paths}/nodes/{node.id}/indices/{index.uuid}/{shard.id}
          */
         public Path resolve(ShardId shardId) {
             return resolve(shardId.getIndex()).resolve(Integer.toString(shardId.id()));
         }
 
         /**
-         * Resolves the given indexes directory against this NodePath
+         * Resolves index directory against this NodePath
+         * ${data.paths}/nodes/{node.id}/indices/{index.uuid}
          */
         public Path resolve(Index index) {
-            return indicesPath.resolve(index.getName());
+            return indicesPath.resolve(index.getUUID());
         }
 
         @Override
@@ -132,25 +132,25 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
 
     private final int localNodeId;
     private final AtomicBoolean closed = new AtomicBoolean(false);
-    private final Map<ShardLockKey, InternalShardLock> shardLocks = new HashMap<>();
+    private final Map<ShardId, InternalShardLock> shardLocks = new HashMap<>();
 
     /**
      * Maximum number of data nodes that should run in an environment.
      */
-    public static final Setting<Integer> MAX_LOCAL_STORAGE_NODES_SETTING = Setting.intSetting("node.max_local_storage_nodes", 50, 1, false,
-            Scope.CLUSTER);
+    public static final Setting<Integer> MAX_LOCAL_STORAGE_NODES_SETTING = Setting.intSetting("node.max_local_storage_nodes", 50, 1,
+            Property.NodeScope);
 
     /**
      * If true automatically append node id to custom data paths.
      */
-    public static final Setting<Boolean> ADD_NODE_ID_TO_CUSTOM_PATH = Setting.boolSetting("node.add_id_to_custom_path", true, false,
-            Scope.CLUSTER);
+    public static final Setting<Boolean> ADD_NODE_ID_TO_CUSTOM_PATH =
+        Setting.boolSetting("node.add_id_to_custom_path", true, Property.NodeScope);
 
     /**
      * If true the [verbose] SegmentInfos.infoStream logging is sent to System.out.
      */
-    public static final Setting<Boolean> ENABLE_LUCENE_SEGMENT_INFOS_TRACE_SETTING = Setting
-            .boolSetting("node.enable_lucene_segment_infos_trace", false, false, Scope.CLUSTER);
+    public static final Setting<Boolean> ENABLE_LUCENE_SEGMENT_INFOS_TRACE_SETTING =
+        Setting.boolSetting("node.enable_lucene_segment_infos_trace", false, Property.NodeScope);
 
     public static final String NODES_FOLDER = "nodes";
     public static final String INDICES_FOLDER = "indices";
@@ -187,7 +187,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
                         logger.trace("obtaining node lock on {} ...", dir.toAbsolutePath());
                         try {
                             locks[dirIndex] = luceneDir.obtainLock(NODE_LOCK_FILENAME);
-                            nodePaths[dirIndex] = new NodePath(dir, environment);
+                            nodePaths[dirIndex] = new NodePath(dir);
                             localNodeId = possibleLockId;
                         } catch (LockObtainFailedException ex) {
                             logger.trace("failed to obtain node lock on {}", dir.toAbsolutePath());
@@ -225,7 +225,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
 
             maybeLogPathDetails();
             maybeLogHeapDetails();
-            
+
             applySegmentInfosTrace(settings);
             assertCanWrite();
             success = true;
@@ -250,7 +250,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
         // We do some I/O in here, so skip this if DEBUG/INFO are not enabled:
         if (logger.isDebugEnabled()) {
             // Log one line per path.data:
-            StringBuilder sb = new StringBuilder("node data locations details:");
+            StringBuilder sb = new StringBuilder();
             for (NodePath nodePath : nodePaths) {
                 sb.append('\n').append(" -> ").append(nodePath.path.toAbsolutePath());
 
@@ -278,7 +278,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
                     .append(fsPath.getType())
                     .append(']');
             }
-            logger.debug(sb.toString());
+            logger.debug("node data locations details:{}", sb);
         } else if (logger.isInfoEnabled()) {
             FsInfo.Path totFSPath = new FsInfo.Path();
             Set<String> allTypes = new HashSet<>();
@@ -306,14 +306,8 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
             }
 
             // Just log a 1-line summary:
-            logger.info(String.format(Locale.ROOT,
-                                      "using [%d] data paths, mounts [%s], net usable_space [%s], net total_space [%s], spins? [%s], types [%s]",
-                                      nodePaths.length,
-                                      allMounts,
-                                      totFSPath.getAvailable(),
-                                      totFSPath.getTotal(),
-                                      toString(allSpins),
-                                      toString(allTypes)));
+            logger.info("using [{}] data paths, mounts [{}], net usable_space [{}], net total_space [{}], spins? [{}], types [{}]",
+                nodePaths.length, allMounts, totFSPath.getAvailable(), totFSPath.getTotal(), toString(allSpins), toString(allTypes));
         }
     }
 
@@ -452,11 +446,11 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
      * @param indexSettings settings for the index being deleted
      */
     public void deleteIndexDirectoryUnderLock(Index index, IndexSettings indexSettings) throws IOException {
-        final Path[] indexPaths = indexPaths(index.getName());
+        final Path[] indexPaths = indexPaths(index);
         logger.trace("deleting index {} directory, paths({}): [{}]", index, indexPaths.length, indexPaths);
         IOUtils.rm(indexPaths);
         if (indexSettings.hasCustomDataPath()) {
-            Path customLocation = resolveCustomLocation(indexSettings, index.getName());
+            Path customLocation = resolveIndexCustomLocation(indexSettings);
             logger.trace("deleting custom index {} directory [{}]", index, customLocation);
             IOUtils.rm(customLocation);
         }
@@ -524,17 +518,16 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
      */
     public ShardLock shardLock(final ShardId shardId, long lockTimeoutMS) throws IOException {
         logger.trace("acquiring node shardlock on [{}], timeout [{}]", shardId, lockTimeoutMS);
-        final ShardLockKey shardLockKey = new ShardLockKey(shardId);
         final InternalShardLock shardLock;
         final boolean acquired;
         synchronized (shardLocks) {
-            if (shardLocks.containsKey(shardLockKey)) {
-                shardLock = shardLocks.get(shardLockKey);
+            if (shardLocks.containsKey(shardId)) {
+                shardLock = shardLocks.get(shardId);
                 shardLock.incWaitCount();
                 acquired = false;
             } else {
-                shardLock = new InternalShardLock(shardLockKey);
-                shardLocks.put(shardLockKey, shardLock);
+                shardLock = new InternalShardLock(shardId);
+                shardLocks.put(shardId, shardLock);
                 acquired = true;
             }
         }
@@ -554,7 +547,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
             @Override
             protected void closeInternal() {
                 shardLock.release();
-                logger.trace("released shard lock for [{}]", shardLockKey);
+                logger.trace("released shard lock for [{}]", shardId);
             }
         };
     }
@@ -566,51 +559,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
      */
     public Set<ShardId> lockedShards() {
         synchronized (shardLocks) {
-            Set<ShardId> lockedShards = shardLocks.keySet().stream()
-                    .map(shardLockKey -> new ShardId(new Index(shardLockKey.indexName, "_na_"), shardLockKey.shardId)).collect(Collectors.toSet());
-            return unmodifiableSet(lockedShards);
-        }
-    }
-
-    // a key for the shard lock. we can't use shardIds, because the contain
-    // the index uuid, but we want the lock semantics to the same as we map indices to disk folders, i.e., without the uuid (for now).
-    private final class ShardLockKey {
-        final String indexName;
-        final int shardId;
-
-        public ShardLockKey(final ShardId shardId) {
-            this.indexName = shardId.getIndexName();
-            this.shardId = shardId.id();
-        }
-
-        @Override
-        public String toString() {
-            return "[" + indexName + "][" + shardId + "]";
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) {
-                return true;
-            }
-            if (o == null || getClass() != o.getClass()) {
-                return false;
-            }
-
-            ShardLockKey that = (ShardLockKey) o;
-
-            if (shardId != that.shardId) {
-                return false;
-            }
-            return indexName.equals(that.indexName);
-
-        }
-
-        @Override
-        public int hashCode() {
-            int result = indexName.hashCode();
-            result = 31 * result + shardId;
-            return result;
+            return unmodifiableSet(new HashSet<>(shardLocks.keySet()));
         }
     }
 
@@ -623,10 +572,10 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
          */
         private final Semaphore mutex = new Semaphore(1);
         private int waitCount = 1; // guarded by shardLocks
-        private final ShardLockKey lockKey;
+        private final ShardId shardId;
 
-        InternalShardLock(ShardLockKey id) {
-            lockKey = id;
+        InternalShardLock(ShardId shardId) {
+            this.shardId = shardId;
             mutex.acquireUninterruptibly();
         }
 
@@ -646,10 +595,10 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
             synchronized (shardLocks) {
                 assert waitCount > 0 : "waitCount is " + waitCount + " but should be > 0";
                 --waitCount;
-                logger.trace("shard lock wait count for [{}] is now [{}]", lockKey, waitCount);
+                logger.trace("shard lock wait count for {} is now [{}]", shardId, waitCount);
                 if (waitCount == 0) {
-                    logger.trace("last shard lock wait decremented, removing lock for [{}]", lockKey);
-                    InternalShardLock remove = shardLocks.remove(lockKey);
+                    logger.trace("last shard lock wait decremented, removing lock for {}", shardId);
+                    InternalShardLock remove = shardLocks.remove(shardId);
                     assert remove != null : "Removed lock was null";
                 }
             }
@@ -658,11 +607,11 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
         void acquire(long timeoutInMillis) throws LockObtainFailedException{
             try {
                 if (mutex.tryAcquire(timeoutInMillis, TimeUnit.MILLISECONDS) == false) {
-                    throw new LockObtainFailedException("Can't lock shard " + lockKey + ", timed out after " + timeoutInMillis + "ms");
+                    throw new LockObtainFailedException("Can't lock shard " + shardId + ", timed out after " + timeoutInMillis + "ms");
                 }
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
-                throw new LockObtainFailedException("Can't lock shard " + lockKey + ", interrupted", e);
+                throw new LockObtainFailedException("Can't lock shard " + shardId + ", interrupted", e);
             }
         }
     }
@@ -705,11 +654,11 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
     /**
      * Returns all index paths.
      */
-    public Path[] indexPaths(String indexName) {
+    public Path[] indexPaths(Index index) {
         assert assertEnvIsLocked();
         Path[] indexPaths = new Path[nodePaths.length];
         for (int i = 0; i < nodePaths.length; i++) {
-            indexPaths[i] = nodePaths[i].indicesPath.resolve(indexName);
+            indexPaths[i] = nodePaths[i].resolve(index);
         }
         return indexPaths;
     }
@@ -732,25 +681,47 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
         return shardLocations;
     }
 
-    public Set<String> findAllIndices() throws IOException {
+    /**
+     * Returns all folder names in ${data.paths}/nodes/{node.id}/indices folder
+     */
+    public Set<String> availableIndexFolders() throws IOException {
         if (nodePaths == null || locks == null) {
             throw new IllegalStateException("node is not configured to store local location");
         }
         assert assertEnvIsLocked();
-        Set<String> indices = new HashSet<>();
+        Set<String> indexFolders = new HashSet<>();
         for (NodePath nodePath : nodePaths) {
             Path indicesLocation = nodePath.indicesPath;
             if (Files.isDirectory(indicesLocation)) {
                 try (DirectoryStream<Path> stream = Files.newDirectoryStream(indicesLocation)) {
                     for (Path index : stream) {
                         if (Files.isDirectory(index)) {
-                            indices.add(index.getFileName().toString());
+                            indexFolders.add(index.getFileName().toString());
                         }
                     }
                 }
             }
         }
-        return indices;
+        return indexFolders;
+
+    }
+
+    /**
+     * Resolves all existing paths to <code>indexFolderName</code> in ${data.paths}/nodes/{node.id}/indices
+     */
+    public Path[] resolveIndexFolder(String indexFolderName) throws IOException {
+        if (nodePaths == null || locks == null) {
+            throw new IllegalStateException("node is not configured to store local location");
+        }
+        assert assertEnvIsLocked();
+        List<Path> paths = new ArrayList<>(nodePaths.length);
+        for (NodePath nodePath : nodePaths) {
+            Path indexFolder = nodePath.indicesPath.resolve(indexFolderName);
+            if (Files.exists(indexFolder)) {
+                paths.add(indexFolder);
+            }
+        }
+        return paths.toArray(new Path[paths.size()]);
     }
 
     /**
@@ -768,13 +739,13 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
         }
         assert assertEnvIsLocked();
         final Set<ShardId> shardIds = new HashSet<>();
-        String indexName = index.getName();
+        final String indexUniquePathId = index.getUUID();
         for (final NodePath nodePath : nodePaths) {
             Path location = nodePath.indicesPath;
             if (Files.isDirectory(location)) {
                 try (DirectoryStream<Path> indexStream = Files.newDirectoryStream(location)) {
                     for (Path indexPath : indexStream) {
-                        if (indexName.equals(indexPath.getFileName().toString())) {
+                        if (indexUniquePathId.equals(indexPath.getFileName().toString())) {
                             shardIds.addAll(findAllShardsForIndex(indexPath, index));
                         }
                     }
@@ -785,7 +756,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
     }
 
     private static Set<ShardId> findAllShardsForIndex(Path indexPath, Index index) throws IOException {
-        assert indexPath.getFileName().toString().equals(index.getName());
+        assert indexPath.getFileName().toString().equals(index.getUUID());
         Set<ShardId> shardIds = new HashSet<>();
         if (Files.isDirectory(indexPath)) {
             try (DirectoryStream<Path> stream = Files.newDirectoryStream(indexPath)) {
@@ -868,7 +839,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
      *
      * @param indexSettings settings for the index
      */
-    private Path resolveCustomLocation(IndexSettings indexSettings) {
+    public Path resolveBaseCustomLocation(IndexSettings indexSettings) {
         String customDataDir = indexSettings.customDataPath();
         if (customDataDir != null) {
             // This assert is because this should be caught by MetaDataCreateIndexService
@@ -889,10 +860,9 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
      * the root path for the index.
      *
      * @param indexSettings settings for the index
-     * @param indexName index to resolve the path for
      */
-    private Path resolveCustomLocation(IndexSettings indexSettings, final String indexName) {
-        return resolveCustomLocation(indexSettings).resolve(indexName);
+    private Path resolveIndexCustomLocation(IndexSettings indexSettings) {
+        return resolveBaseCustomLocation(indexSettings).resolve(indexSettings.getUUID());
     }
 
     /**
@@ -904,7 +874,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
      * @param shardId shard to resolve the path to
      */
     public Path resolveCustomLocation(IndexSettings indexSettings, final ShardId shardId) {
-        return resolveCustomLocation(indexSettings, shardId.getIndexName()).resolve(Integer.toString(shardId.id()));
+        return resolveIndexCustomLocation(indexSettings).resolve(Integer.toString(shardId.id()));
     }
 
     /**
@@ -928,22 +898,24 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
         for (Path path : nodeDataPaths()) { // check node-paths are writable
             tryWriteTempFile(path);
         }
-        for (String index : this.findAllIndices()) {
-            for (Path path : this.indexPaths(index)) { // check index paths are writable
-                Path statePath = path.resolve(MetaDataStateFormat.STATE_DIR_NAME);
-                tryWriteTempFile(statePath);
-                tryWriteTempFile(path);
-            }
-            for (ShardId shardID : this.findAllShardIds(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE))) {
-                Path[] paths = this.availableShardPaths(shardID);
-                for (Path path : paths) { // check shard paths are writable
-                    Path indexDir = path.resolve(ShardPath.INDEX_FOLDER_NAME);
-                    Path statePath = path.resolve(MetaDataStateFormat.STATE_DIR_NAME);
-                    Path translogDir = path.resolve(ShardPath.TRANSLOG_FOLDER_NAME);
-                    tryWriteTempFile(indexDir);
-                    tryWriteTempFile(translogDir);
-                    tryWriteTempFile(statePath);
-                    tryWriteTempFile(path);
+        for (String indexFolderName : this.availableIndexFolders()) {
+            for (Path indexPath : this.resolveIndexFolder(indexFolderName)) { // check index paths are writable
+                Path indexStatePath = indexPath.resolve(MetaDataStateFormat.STATE_DIR_NAME);
+                tryWriteTempFile(indexStatePath);
+                tryWriteTempFile(indexPath);
+                try (DirectoryStream<Path> stream = Files.newDirectoryStream(indexPath)) {
+                    for (Path shardPath : stream) {
+                        String fileName = shardPath.getFileName().toString();
+                        if (Files.isDirectory(shardPath) && fileName.chars().allMatch(Character::isDigit)) {
+                            Path indexDir = shardPath.resolve(ShardPath.INDEX_FOLDER_NAME);
+                            Path statePath = shardPath.resolve(MetaDataStateFormat.STATE_DIR_NAME);
+                            Path translogDir = shardPath.resolve(ShardPath.TRANSLOG_FOLDER_NAME);
+                            tryWriteTempFile(indexDir);
+                            tryWriteTempFile(translogDir);
+                            tryWriteTempFile(statePath);
+                            tryWriteTempFile(shardPath);
+                        }
+                    }
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java b/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
index 757a78c3b5f..1ccdb43cc45 100644
--- a/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
+++ b/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
@@ -269,7 +269,7 @@ public abstract class AsyncShardFetch<T extends BaseNodeResponse> implements Rel
      */
     // visible for testing
     void asyncFetch(final ShardId shardId, final String[] nodesIds, final MetaData metaData) {
-        IndexMetaData indexMetaData = metaData.index(shardId.getIndex());
+        IndexMetaData indexMetaData = metaData.getIndexSafe(shardId.getIndex());
         logger.trace("{} fetching [{}] from {}", shardId, type, nodesIds);
         action.list(shardId, indexMetaData, nodesIds, new ActionListener<BaseNodesResponse<T>>() {
             @Override
diff --git a/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java b/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
index e2fcb56b1e1..b4d8eeae532 100644
--- a/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
+++ b/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.gateway;
 
+import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.component.AbstractComponent;
@@ -26,12 +27,17 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.Index;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.unmodifiableMap;
@@ -47,7 +53,7 @@ public class DanglingIndicesState extends AbstractComponent {
     private final MetaStateService metaStateService;
     private final LocalAllocateDangledIndices allocateDangledIndices;
 
-    private final Map<String, IndexMetaData> danglingIndices = ConcurrentCollections.newConcurrentMap();
+    private final Map<Index, IndexMetaData> danglingIndices = ConcurrentCollections.newConcurrentMap();
 
     @Inject
     public DanglingIndicesState(Settings settings, NodeEnvironment nodeEnv, MetaStateService metaStateService,
@@ -74,7 +80,7 @@ public class DanglingIndicesState extends AbstractComponent {
     /**
      * The current set of dangling indices.
      */
-    Map<String, IndexMetaData> getDanglingIndices() {
+    Map<Index, IndexMetaData> getDanglingIndices() {
         // This might be a good use case for CopyOnWriteHashMap
         return unmodifiableMap(new HashMap<>(danglingIndices));
     }
@@ -83,10 +89,16 @@ public class DanglingIndicesState extends AbstractComponent {
      * Cleans dangling indices if they are already allocated on the provided meta data.
      */
     void cleanupAllocatedDangledIndices(MetaData metaData) {
-        for (String danglingIndex : danglingIndices.keySet()) {
-            if (metaData.hasIndex(danglingIndex)) {
-                logger.debug("[{}] no longer dangling (created), removing from dangling list", danglingIndex);
-                danglingIndices.remove(danglingIndex);
+        for (Index index : danglingIndices.keySet()) {
+            final IndexMetaData indexMetaData = metaData.index(index);
+            if (indexMetaData != null && indexMetaData.getIndex().getName().equals(index.getName())) {
+                if (indexMetaData.getIndex().getUUID().equals(index.getUUID()) == false) {
+                    logger.warn("[{}] can not be imported as a dangling index, as there is already another index " +
+                        "with the same name but a different uuid. local index will be ignored (but not deleted)", index);
+                } else {
+                    logger.debug("[{}] no longer dangling (created), removing from dangling list", index);
+                }
+                danglingIndices.remove(index);
             }
         }
     }
@@ -104,36 +116,30 @@ public class DanglingIndicesState extends AbstractComponent {
      * that have state on disk, but are not part of the provided meta data, or not detected
      * as dangled already.
      */
-    Map<String, IndexMetaData> findNewDanglingIndices(MetaData metaData) {
-        final Set<String> indices;
+    Map<Index, IndexMetaData> findNewDanglingIndices(MetaData metaData) {
+        final Set<String> excludeIndexPathIds = new HashSet<>(metaData.indices().size() + danglingIndices.size());
+        for (ObjectCursor<IndexMetaData> cursor : metaData.indices().values()) {
+            excludeIndexPathIds.add(cursor.value.getIndex().getUUID());
+        }
+        excludeIndexPathIds.addAll(danglingIndices.keySet().stream().map(Index::getUUID).collect(Collectors.toList()));
         try {
-            indices = nodeEnv.findAllIndices();
-        } catch (Throwable e) {
+            final List<IndexMetaData> indexMetaDataList = metaStateService.loadIndicesStates(excludeIndexPathIds::contains);
+            Map<Index, IndexMetaData> newIndices = new HashMap<>(indexMetaDataList.size());
+            for (IndexMetaData indexMetaData : indexMetaDataList) {
+                if (metaData.hasIndex(indexMetaData.getIndex().getName())) {
+                    logger.warn("[{}] can not be imported as a dangling index, as index with same name already exists in cluster metadata",
+                        indexMetaData.getIndex());
+                } else {
+                    logger.info("[{}] dangling index, exists on local file system, but not in cluster metadata, auto import to cluster state",
+                        indexMetaData.getIndex());
+                    newIndices.put(indexMetaData.getIndex(), indexMetaData);
+                }
+            }
+            return newIndices;
+        } catch (IOException e) {
             logger.warn("failed to list dangling indices", e);
             return emptyMap();
         }
-
-        Map<String, IndexMetaData>  newIndices = new HashMap<>();
-        for (String indexName : indices) {
-            if (metaData.hasIndex(indexName) == false && danglingIndices.containsKey(indexName) == false) {
-                try {
-                    IndexMetaData indexMetaData = metaStateService.loadIndexState(indexName);
-                    if (indexMetaData != null) {
-                        logger.info("[{}] dangling index, exists on local file system, but not in cluster metadata, auto import to cluster state", indexName);
-                        if (!indexMetaData.getIndex().getName().equals(indexName)) {
-                            logger.info("dangled index directory name is [{}], state name is [{}], renaming to directory name", indexName, indexMetaData.getIndex());
-                            indexMetaData = IndexMetaData.builder(indexMetaData).index(indexName).build();
-                        }
-                        newIndices.put(indexName, indexMetaData);
-                    } else {
-                        logger.debug("[{}] dangling index directory detected, but no state found", indexName);
-                    }
-                } catch (Throwable t) {
-                    logger.warn("[{}] failed to load index state for detected dangled index", t, indexName);
-                }
-            }
-        }
-        return newIndices;
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/gateway/Gateway.java b/core/src/main/java/org/elasticsearch/gateway/Gateway.java
index fd3bd9a0b6d..b2cb2d11079 100644
--- a/core/src/main/java/org/elasticsearch/gateway/Gateway.java
+++ b/core/src/main/java/org/elasticsearch/gateway/Gateway.java
@@ -20,22 +20,26 @@
 package org.elasticsearch.gateway;
 
 import com.carrotsearch.hppc.ObjectFloatHashMap;
-import com.carrotsearch.hppc.ObjectHashSet;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.NodeServicesProvider;
+import org.elasticsearch.indices.IndicesService;
 
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.function.Supplier;
 
 /**
@@ -52,10 +56,15 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
     private final TransportNodesListGatewayMetaState listGatewayMetaState;
 
     private final Supplier<Integer> minimumMasterNodesProvider;
+    private final IndicesService indicesService;
+    private final NodeServicesProvider nodeServicesProvider;
 
     public Gateway(Settings settings, ClusterService clusterService, NodeEnvironment nodeEnv, GatewayMetaState metaState,
-                   TransportNodesListGatewayMetaState listGatewayMetaState, Discovery discovery) {
+                   TransportNodesListGatewayMetaState listGatewayMetaState, Discovery discovery,
+                   NodeServicesProvider nodeServicesProvider, IndicesService indicesService) {
         super(settings);
+        this.nodeServicesProvider = nodeServicesProvider;
+        this.indicesService = indicesService;
         this.clusterService = clusterService;
         this.nodeEnv = nodeEnv;
         this.metaState = metaState;
@@ -65,9 +74,9 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
     }
 
     public void performStateRecovery(final GatewayStateRecoveredListener listener) throws GatewayException {
-        ObjectHashSet<String> nodesIds = new ObjectHashSet<>(clusterService.state().nodes().masterNodes().keys());
-        logger.trace("performing state recovery from {}", nodesIds);
-        TransportNodesListGatewayMetaState.NodesGatewayMetaState nodesState = listGatewayMetaState.list(nodesIds.toArray(String.class), null).actionGet();
+        String[] nodesIds = clusterService.state().nodes().masterNodes().keys().toArray(String.class);
+        logger.trace("performing state recovery from {}", Arrays.toString(nodesIds));
+        TransportNodesListGatewayMetaState.NodesGatewayMetaState nodesState = listGatewayMetaState.list(nodesIds, null).actionGet();
 
 
         int requiredAllocation = Math.max(1, minimumMasterNodesProvider.get());
@@ -79,7 +88,7 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
             }
         }
 
-        ObjectFloatHashMap<String> indices = new ObjectFloatHashMap<>();
+        ObjectFloatHashMap<Index> indices = new ObjectFloatHashMap<>();
         MetaData electedGlobalState = null;
         int found = 0;
         for (TransportNodesListGatewayMetaState.NodeGatewayMetaState nodeState : nodesState) {
@@ -93,7 +102,7 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
                 electedGlobalState = nodeState.metaData();
             }
             for (ObjectCursor<IndexMetaData> cursor : nodeState.metaData().indices().values()) {
-                indices.addTo(cursor.value.getIndex().getName(), 1);
+                indices.addTo(cursor.value.getIndex(), 1);
             }
         }
         if (found < requiredAllocation) {
@@ -107,7 +116,7 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
         final Object[] keys = indices.keys;
         for (int i = 0; i < keys.length; i++) {
             if (keys[i] != null) {
-                String index = (String) keys[i];
+                Index index = (Index) keys[i];
                 IndexMetaData electedIndexMetaData = null;
                 int indexMetaDataCount = 0;
                 for (TransportNodesListGatewayMetaState.NodeGatewayMetaState nodeState : nodesState) {
@@ -128,11 +137,24 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
                 if (electedIndexMetaData != null) {
                     if (indexMetaDataCount < requiredAllocation) {
                         logger.debug("[{}] found [{}], required [{}], not adding", index, indexMetaDataCount, requiredAllocation);
+                    } // TODO if this logging statement is correct then we are missing an else here
+                    try {
+                        if (electedIndexMetaData.getState() == IndexMetaData.State.OPEN) {
+                            // verify that we can actually create this index - if not we recover it as closed with lots of warn logs
+                            indicesService.verifyIndexMetadata(nodeServicesProvider, electedIndexMetaData);
+                        }
+                    } catch (Exception e) {
+                        logger.warn("recovering index {} failed - recovering as closed", e, electedIndexMetaData.getIndex());
+                        electedIndexMetaData = IndexMetaData.builder(electedIndexMetaData).state(IndexMetaData.State.CLOSE).build();
                     }
+
                     metaDataBuilder.put(electedIndexMetaData, false);
                 }
             }
         }
+        final ClusterSettings clusterSettings = clusterService.getClusterSettings();
+        metaDataBuilder.persistentSettings(clusterSettings.archiveUnknownOrBrokenSettings(metaDataBuilder.persistentSettings()));
+        metaDataBuilder.transientSettings(clusterSettings.archiveUnknownOrBrokenSettings(metaDataBuilder.transientSettings()));
         ClusterState.Builder builder = ClusterState.builder(clusterService.state().getClusterName());
         builder.metaData(metaDataBuilder);
         listener.onSuccess(builder.build());
diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java b/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
index acd650bc6f7..0059a0ef61b 100644
--- a/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
@@ -22,7 +22,6 @@ package org.elasticsearch.gateway;
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
@@ -31,6 +30,7 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lease.Releasables;
diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
index e90cb750cf5..867b7420107 100644
--- a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
+++ b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
@@ -34,7 +34,9 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.IndexFolderUpgrader;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.Index;
 
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
@@ -61,7 +63,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
     @Nullable
     private volatile MetaData previousMetaData;
 
-    private volatile Set<String> previouslyWrittenIndices = emptySet();
+    private volatile Set<Index> previouslyWrittenIndices = emptySet();
 
     @Inject
     public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateService metaStateService,
@@ -84,7 +86,8 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
         if (DiscoveryNode.masterNode(settings) || DiscoveryNode.dataNode(settings)) {
             try {
                 ensureNoPre019State();
-                pre20Upgrade();
+                IndexFolderUpgrader.upgradeIndicesIfNeeded(settings, nodeEnv);
+                upgradeMetaData();
                 long startNS = System.nanoTime();
                 metaStateService.loadFullState();
                 logger.debug("took {} to load state", TimeValue.timeValueMillis(TimeValue.nsecToMSec(System.nanoTime() - startNS)));
@@ -102,7 +105,6 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
     @Override
     public void clusterChanged(ClusterChangedEvent event) {
 
-        Set<String> relevantIndices = new HashSet<>();
         final ClusterState state = event.state();
         if (state.blocks().disableStatePersistence()) {
             // reset the current metadata, we need to start fresh...
@@ -113,7 +115,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
 
         MetaData newMetaData = state.metaData();
         // we don't check if metaData changed, since we might be called several times and we need to check dangling...
-
+        Set<Index> relevantIndices = Collections.emptySet();
         boolean success = true;
         // write the state if this node is a master eligible node or if it is a data node and has shards allocated on it
         if (state.nodes().localNode().masterNode() || state.nodes().localNode().dataNode()) {
@@ -126,14 +128,14 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
                     // persistence was disabled or the node was restarted), see getRelevantIndicesOnDataOnlyNode().
                     // we therefore have to check here if we have shards on disk and add their indices to the previouslyWrittenIndices list
                     if (isDataOnlyNode(state)) {
-                        Set<String> newPreviouslyWrittenIndices = new HashSet<>(previouslyWrittenIndices.size());
+                        Set<Index> newPreviouslyWrittenIndices = new HashSet<>(previouslyWrittenIndices.size());
                         for (IndexMetaData indexMetaData : newMetaData) {
                             IndexMetaData indexMetaDataOnDisk = null;
                             if (indexMetaData.getState().equals(IndexMetaData.State.CLOSE)) {
-                                indexMetaDataOnDisk = metaStateService.loadIndexState(indexMetaData.getIndex().getName());
+                                indexMetaDataOnDisk = metaStateService.loadIndexState(indexMetaData.getIndex());
                             }
                             if (indexMetaDataOnDisk != null) {
-                                newPreviouslyWrittenIndices.add(indexMetaDataOnDisk.getIndex().getName());
+                                newPreviouslyWrittenIndices.add(indexMetaDataOnDisk.getIndex());
                             }
                         }
                         newPreviouslyWrittenIndices.addAll(previouslyWrittenIndices);
@@ -152,13 +154,13 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
                 }
             }
 
-            Iterable<IndexMetaWriteInfo> writeInfo;
+
             relevantIndices = getRelevantIndices(event.state(), event.previousState(), previouslyWrittenIndices);
-            writeInfo = resolveStatesToBeWritten(previouslyWrittenIndices, relevantIndices, previousMetaData, event.state().metaData());
+            final Iterable<IndexMetaWriteInfo> writeInfo = resolveStatesToBeWritten(previouslyWrittenIndices, relevantIndices, previousMetaData, event.state().metaData());
             // check and write changes in indices
             for (IndexMetaWriteInfo indexMetaWrite : writeInfo) {
                 try {
-                    metaStateService.writeIndex(indexMetaWrite.reason, indexMetaWrite.newMetaData, indexMetaWrite.previousMetaData);
+                    metaStateService.writeIndex(indexMetaWrite.reason, indexMetaWrite.newMetaData);
                 } catch (Throwable e) {
                     success = false;
                 }
@@ -166,15 +168,14 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
         }
 
         danglingIndicesState.processDanglingIndices(newMetaData);
-
         if (success) {
             previousMetaData = newMetaData;
             previouslyWrittenIndices = unmodifiableSet(relevantIndices);
         }
     }
 
-    public static Set<String> getRelevantIndices(ClusterState state, ClusterState previousState, Set<String> previouslyWrittenIndices) {
-        Set<String> relevantIndices;
+    public static Set<Index> getRelevantIndices(ClusterState state, ClusterState previousState, Set<Index> previouslyWrittenIndices) {
+        Set<Index> relevantIndices;
         if (isDataOnlyNode(state)) {
             relevantIndices = getRelevantIndicesOnDataOnlyNode(state, previousState, previouslyWrittenIndices);
         } else if (state.nodes().localNode().masterNode() == true) {
@@ -202,7 +203,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
             try (DirectoryStream<Path> stream = Files.newDirectoryStream(stateLocation)) {
                 for (Path stateFile : stream) {
                     if (logger.isTraceEnabled()) {
-                        logger.trace("[upgrade]: processing [" + stateFile.getFileName() + "]");
+                        logger.trace("[upgrade]: processing [{}]", stateFile.getFileName());
                     }
                     final String name = stateFile.getFileName().toString();
                     if (name.startsWith("metadata-")) {
@@ -221,7 +222,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
      * MetaDataIndexUpgradeService might also update obsolete settings if needed. When this happens we rewrite
      * index metadata with new settings.
      */
-    private void pre20Upgrade() throws Exception {
+    private void upgradeMetaData() throws Exception {
         MetaData metaData = loadMetaState();
         List<IndexMetaData> updateIndexMetaData = new ArrayList<>();
         for (IndexMetaData indexMetaData : metaData) {
@@ -233,7 +234,8 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
         // We successfully checked all indices for backward compatibility and found no non-upgradable indices, which
         // means the upgrade can continue. Now it's safe to overwrite index metadata with the new version.
         for (IndexMetaData indexMetaData : updateIndexMetaData) {
-            metaStateService.writeIndex("upgrade", indexMetaData, null);
+            // since we upgraded the index folders already, write index state in the upgraded index folder
+            metaStateService.writeIndex("upgrade", indexMetaData);
         }
     }
 
@@ -264,10 +266,10 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
      * @param newMetaData                 The new metadata
      * @return iterable over all indices states that should be written to disk
      */
-    public static Iterable<GatewayMetaState.IndexMetaWriteInfo> resolveStatesToBeWritten(Set<String> previouslyWrittenIndices, Set<String> potentiallyUnwrittenIndices, MetaData previousMetaData, MetaData newMetaData) {
+    public static Iterable<GatewayMetaState.IndexMetaWriteInfo> resolveStatesToBeWritten(Set<Index> previouslyWrittenIndices, Set<Index> potentiallyUnwrittenIndices, MetaData previousMetaData, MetaData newMetaData) {
         List<GatewayMetaState.IndexMetaWriteInfo> indicesToWrite = new ArrayList<>();
-        for (String index : potentiallyUnwrittenIndices) {
-            IndexMetaData newIndexMetaData = newMetaData.index(index);
+        for (Index index : potentiallyUnwrittenIndices) {
+            IndexMetaData newIndexMetaData = newMetaData.getIndexSafe(index);
             IndexMetaData previousIndexMetaData = previousMetaData == null ? null : previousMetaData.index(index);
             String writeReason = null;
             if (previouslyWrittenIndices.contains(index) == false || previousIndexMetaData == null) {
@@ -282,14 +284,14 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
         return indicesToWrite;
     }
 
-    public static Set<String> getRelevantIndicesOnDataOnlyNode(ClusterState state, ClusterState previousState, Set<String> previouslyWrittenIndices) {
+    public static Set<Index> getRelevantIndicesOnDataOnlyNode(ClusterState state, ClusterState previousState, Set<Index> previouslyWrittenIndices) {
         RoutingNode newRoutingNode = state.getRoutingNodes().node(state.nodes().localNodeId());
         if (newRoutingNode == null) {
             throw new IllegalStateException("cluster state does not contain this node - cannot write index meta state");
         }
-        Set<String> indices = new HashSet<>();
+        Set<Index> indices = new HashSet<>();
         for (ShardRouting routing : newRoutingNode) {
-            indices.add(routing.index().getName());
+            indices.add(routing.index());
         }
         // we have to check the meta data also: closed indices will not appear in the routing table, but we must still write the state if we have it written on disk previously
         for (IndexMetaData indexMetaData : state.metaData()) {
@@ -300,19 +302,19 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
             if (previousMetaData != null) {
                 isOrWasClosed = isOrWasClosed || previousMetaData.getState().equals(IndexMetaData.State.CLOSE);
             }
-            if (previouslyWrittenIndices.contains(indexMetaData.getIndex().getName()) && isOrWasClosed) {
-                indices.add(indexMetaData.getIndex().getName());
+            if (previouslyWrittenIndices.contains(indexMetaData.getIndex()) && isOrWasClosed) {
+                indices.add(indexMetaData.getIndex());
             }
         }
         return indices;
     }
 
-    public static Set<String> getRelevantIndicesForMasterEligibleNode(ClusterState state) {
-        Set<String> relevantIndices;
+    public static Set<Index> getRelevantIndicesForMasterEligibleNode(ClusterState state) {
+        Set<Index> relevantIndices;
         relevantIndices = new HashSet<>();
         // we have to iterate over the metadata to make sure we also capture closed indices
         for (IndexMetaData indexMetaData : state.metaData()) {
-            relevantIndices.add(indexMetaData.getIndex().getName());
+            relevantIndices.add(indexMetaData.getIndex());
         }
         return relevantIndices;
     }
diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayService.java b/core/src/main/java/org/elasticsearch/gateway/GatewayService.java
index 384539b4c63..7dcc45f1c0a 100644
--- a/core/src/main/java/org/elasticsearch/gateway/GatewayService.java
+++ b/core/src/main/java/org/elasticsearch/gateway/GatewayService.java
@@ -21,7 +21,6 @@ package org.elasticsearch.gateway;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
@@ -34,14 +33,18 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.NodeServicesProvider;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -52,20 +55,20 @@ import java.util.concurrent.atomic.AtomicBoolean;
  */
 public class GatewayService extends AbstractLifecycleComponent<GatewayService> implements ClusterStateListener {
 
-    public static final Setting<Integer> EXPECTED_NODES_SETTING = Setting.intSetting(
-            "gateway.expected_nodes", -1, -1, false, Setting.Scope.CLUSTER);
-    public static final Setting<Integer> EXPECTED_DATA_NODES_SETTING = Setting.intSetting(
-            "gateway.expected_data_nodes", -1, -1, false, Setting.Scope.CLUSTER);
-    public static final Setting<Integer> EXPECTED_MASTER_NODES_SETTING = Setting.intSetting(
-            "gateway.expected_master_nodes", -1, -1, false, Setting.Scope.CLUSTER);
-    public static final Setting<TimeValue> RECOVER_AFTER_TIME_SETTING = Setting.positiveTimeSetting(
-            "gateway.recover_after_time", TimeValue.timeValueMillis(0), false, Setting.Scope.CLUSTER);
-    public static final Setting<Integer> RECOVER_AFTER_NODES_SETTING = Setting.intSetting(
-            "gateway.recover_after_nodes", -1, -1, false, Setting.Scope.CLUSTER);
-    public static final Setting<Integer> RECOVER_AFTER_DATA_NODES_SETTING = Setting.intSetting(
-            "gateway.recover_after_data_nodes", -1, -1, false, Setting.Scope.CLUSTER);
-    public static final Setting<Integer> RECOVER_AFTER_MASTER_NODES_SETTING = Setting.intSetting(
-            "gateway.recover_after_master_nodes", 0, 0, false, Setting.Scope.CLUSTER);
+    public static final Setting<Integer> EXPECTED_NODES_SETTING =
+        Setting.intSetting("gateway.expected_nodes", -1, -1, Property.NodeScope);
+    public static final Setting<Integer> EXPECTED_DATA_NODES_SETTING =
+        Setting.intSetting("gateway.expected_data_nodes", -1, -1, Property.NodeScope);
+    public static final Setting<Integer> EXPECTED_MASTER_NODES_SETTING =
+        Setting.intSetting("gateway.expected_master_nodes", -1, -1, Property.NodeScope);
+    public static final Setting<TimeValue> RECOVER_AFTER_TIME_SETTING =
+        Setting.positiveTimeSetting("gateway.recover_after_time", TimeValue.timeValueMillis(0), Property.NodeScope);
+    public static final Setting<Integer> RECOVER_AFTER_NODES_SETTING =
+        Setting.intSetting("gateway.recover_after_nodes", -1, -1, Property.NodeScope);
+    public static final Setting<Integer> RECOVER_AFTER_DATA_NODES_SETTING =
+        Setting.intSetting("gateway.recover_after_data_nodes", -1, -1, Property.NodeScope);
+    public static final Setting<Integer> RECOVER_AFTER_MASTER_NODES_SETTING =
+        Setting.intSetting("gateway.recover_after_master_nodes", 0, 0, Property.NodeScope);
 
     public static final ClusterBlock STATE_NOT_RECOVERED_BLOCK = new ClusterBlock(1, "state not recovered / initialized", true, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL);
 
@@ -94,9 +97,11 @@ public class GatewayService extends AbstractLifecycleComponent<GatewayService> i
     @Inject
     public GatewayService(Settings settings, AllocationService allocationService, ClusterService clusterService,
                           ThreadPool threadPool, NodeEnvironment nodeEnvironment, GatewayMetaState metaState,
-                          TransportNodesListGatewayMetaState listGatewayMetaState, Discovery discovery) {
+                          TransportNodesListGatewayMetaState listGatewayMetaState, Discovery discovery,
+                          NodeServicesProvider nodeServicesProvider, IndicesService indicesService) {
         super(settings);
-        this.gateway = new Gateway(settings, clusterService, nodeEnvironment, metaState, listGatewayMetaState, discovery);
+        this.gateway = new Gateway(settings, clusterService, nodeEnvironment, metaState, listGatewayMetaState, discovery,
+            nodeServicesProvider, indicesService);
         this.allocationService = allocationService;
         this.clusterService = clusterService;
         this.threadPool = threadPool;
@@ -161,11 +166,14 @@ public class GatewayService extends AbstractLifecycleComponent<GatewayService> i
         if (state.nodes().masterNodeId() == null) {
             logger.debug("not recovering from gateway, no master elected yet");
         } else if (recoverAfterNodes != -1 && (nodes.masterAndDataNodes().size()) < recoverAfterNodes) {
-            logger.debug("not recovering from gateway, nodes_size (data+master) [" + nodes.masterAndDataNodes().size() + "] < recover_after_nodes [" + recoverAfterNodes + "]");
+            logger.debug("not recovering from gateway, nodes_size (data+master) [{}] < recover_after_nodes [{}]",
+                nodes.masterAndDataNodes().size(), recoverAfterNodes);
         } else if (recoverAfterDataNodes != -1 && nodes.dataNodes().size() < recoverAfterDataNodes) {
-            logger.debug("not recovering from gateway, nodes_size (data) [" + nodes.dataNodes().size() + "] < recover_after_data_nodes [" + recoverAfterDataNodes + "]");
+            logger.debug("not recovering from gateway, nodes_size (data) [{}] < recover_after_data_nodes [{}]",
+                nodes.dataNodes().size(), recoverAfterDataNodes);
         } else if (recoverAfterMasterNodes != -1 && nodes.masterNodes().size() < recoverAfterMasterNodes) {
-            logger.debug("not recovering from gateway, nodes_size (master) [" + nodes.masterNodes().size() + "] < recover_after_master_nodes [" + recoverAfterMasterNodes + "]");
+            logger.debug("not recovering from gateway, nodes_size (master) [{}] < recover_after_master_nodes [{}]",
+                nodes.masterNodes().size(), recoverAfterMasterNodes);
         } else {
             boolean enforceRecoverAfterTime;
             String reason;
diff --git a/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java b/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
index 041b8cafecc..b14dcc6d1a4 100644
--- a/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
+++ b/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.gateway;
 
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlocks;
@@ -30,6 +29,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
diff --git a/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java
index 9ef09753c43..89192c47d09 100644
--- a/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java
+++ b/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java
@@ -25,58 +25,26 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Predicate;
 
 /**
  * Handles writing and loading both {@link MetaData} and {@link IndexMetaData}
  */
 public class MetaStateService extends AbstractComponent {
 
-    static final String FORMAT_SETTING = "gateway.format";
-
-    static final String GLOBAL_STATE_FILE_PREFIX = "global-";
-    private static final String INDEX_STATE_FILE_PREFIX = "state-";
-
     private final NodeEnvironment nodeEnv;
 
-    private final XContentType format;
-    private final ToXContent.Params formatParams;
-    private final ToXContent.Params gatewayModeFormatParams;
-    private final MetaDataStateFormat<IndexMetaData> indexStateFormat;
-    private final MetaDataStateFormat<MetaData> globalStateFormat;
-
     @Inject
     public MetaStateService(Settings settings, NodeEnvironment nodeEnv) {
         super(settings);
         this.nodeEnv = nodeEnv;
-        this.format = XContentType.fromMediaTypeOrFormat(settings.get(FORMAT_SETTING, "smile"));
-        if (this.format == XContentType.SMILE) {
-            Map<String, String> params = new HashMap<>();
-            params.put("binary", "true");
-            formatParams = new ToXContent.MapParams(params);
-            Map<String, String> gatewayModeParams = new HashMap<>();
-            gatewayModeParams.put("binary", "true");
-            gatewayModeParams.put(MetaData.CONTEXT_MODE_PARAM, MetaData.CONTEXT_MODE_GATEWAY);
-            gatewayModeFormatParams = new ToXContent.MapParams(gatewayModeParams);
-        } else {
-            formatParams = ToXContent.EMPTY_PARAMS;
-            Map<String, String> gatewayModeParams = new HashMap<>();
-            gatewayModeParams.put(MetaData.CONTEXT_MODE_PARAM, MetaData.CONTEXT_MODE_GATEWAY);
-            gatewayModeFormatParams = new ToXContent.MapParams(gatewayModeParams);
-        }
-        indexStateFormat = indexStateFormat(format, formatParams);
-        globalStateFormat = globalStateFormat(format, gatewayModeFormatParams);
-
     }
 
     /**
@@ -91,14 +59,12 @@ public class MetaStateService extends AbstractComponent {
         } else {
             metaDataBuilder = MetaData.builder();
         }
-
-        final Set<String> indices = nodeEnv.findAllIndices();
-        for (String index : indices) {
-            IndexMetaData indexMetaData = loadIndexState(index);
-            if (indexMetaData == null) {
-                logger.debug("[{}] failed to find metadata for existing index location", index);
-            } else {
+        for (String indexFolderName : nodeEnv.availableIndexFolders()) {
+            IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, nodeEnv.resolveIndexFolder(indexFolderName));
+            if (indexMetaData != null) {
                 metaDataBuilder.put(indexMetaData, false);
+            } else {
+                logger.debug("[{}] failed to find metadata for existing index location", indexFolderName);
             }
         }
         return metaDataBuilder.build();
@@ -108,15 +74,40 @@ public class MetaStateService extends AbstractComponent {
      * Loads the index state for the provided index name, returning null if doesn't exists.
      */
     @Nullable
-    IndexMetaData loadIndexState(String index) throws IOException {
-        return indexStateFormat.loadLatestState(logger, nodeEnv.indexPaths(index));
+    IndexMetaData loadIndexState(Index index) throws IOException {
+        return IndexMetaData.FORMAT.loadLatestState(logger, nodeEnv.indexPaths(index));
+    }
+
+    /**
+     * Loads all indices states available on disk
+     */
+    List<IndexMetaData> loadIndicesStates(Predicate<String> excludeIndexPathIdsPredicate) throws IOException {
+        List<IndexMetaData> indexMetaDataList = new ArrayList<>();
+        for (String indexFolderName : nodeEnv.availableIndexFolders()) {
+            if (excludeIndexPathIdsPredicate.test(indexFolderName)) {
+                continue;
+            }
+            IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger,
+                nodeEnv.resolveIndexFolder(indexFolderName));
+            if (indexMetaData != null) {
+                final String indexPathId = indexMetaData.getIndex().getUUID();
+                if (indexFolderName.equals(indexPathId)) {
+                    indexMetaDataList.add(indexMetaData);
+                } else {
+                    throw new IllegalStateException("[" + indexFolderName+ "] invalid index folder name, rename to [" + indexPathId + "]");
+                }
+            } else {
+                logger.debug("[{}] failed to find metadata for existing index location", indexFolderName);
+            }
+        }
+        return indexMetaDataList;
     }
 
     /**
      * Loads the global state, *without* index state, see {@link #loadFullState()} for that.
      */
     MetaData loadGlobalState() throws IOException {
-        MetaData globalState = globalStateFormat.loadLatestState(logger, nodeEnv.nodeDataPaths());
+        MetaData globalState = MetaData.FORMAT.loadLatestState(logger, nodeEnv.nodeDataPaths());
         // ES 2.0 now requires units for all time and byte-sized settings, so we add the default unit if it's missing
         // TODO: can we somehow only do this for pre-2.0 cluster state?
         if (globalState != null) {
@@ -129,13 +120,15 @@ public class MetaStateService extends AbstractComponent {
     /**
      * Writes the index state.
      */
-    void writeIndex(String reason, IndexMetaData indexMetaData, @Nullable IndexMetaData previousIndexMetaData) throws Exception {
-        logger.trace("[{}] writing state, reason [{}]", indexMetaData.getIndex(), reason);
+    void writeIndex(String reason, IndexMetaData indexMetaData) throws IOException {
+        final Index index = indexMetaData.getIndex();
+        logger.trace("[{}] writing state, reason [{}]", index, reason);
         try {
-            indexStateFormat.write(indexMetaData, indexMetaData.getVersion(), nodeEnv.indexPaths(indexMetaData.getIndex().getName()));
+            IndexMetaData.FORMAT.write(indexMetaData, indexMetaData.getVersion(),
+                nodeEnv.indexPaths(indexMetaData.getIndex()));
         } catch (Throwable ex) {
-            logger.warn("[{}]: failed to write index state", ex, indexMetaData.getIndex());
-            throw new IOException("failed to write state for [" + indexMetaData.getIndex() + "]", ex);
+            logger.warn("[{}]: failed to write index state", ex, index);
+            throw new IOException("failed to write state for [" + index + "]", ex);
         }
     }
 
@@ -145,45 +138,10 @@ public class MetaStateService extends AbstractComponent {
     void writeGlobalState(String reason, MetaData metaData) throws Exception {
         logger.trace("[_global] writing state, reason [{}]",  reason);
         try {
-            globalStateFormat.write(metaData, metaData.version(), nodeEnv.nodeDataPaths());
+            MetaData.FORMAT.write(metaData, metaData.version(), nodeEnv.nodeDataPaths());
         } catch (Throwable ex) {
             logger.warn("[_global]: failed to write global state", ex);
             throw new IOException("failed to write global state", ex);
         }
     }
-
-    /**
-     * Returns a StateFormat that can read and write {@link MetaData}
-     */
-    static MetaDataStateFormat<MetaData> globalStateFormat(XContentType format, final ToXContent.Params formatParams) {
-        return new MetaDataStateFormat<MetaData>(format, GLOBAL_STATE_FILE_PREFIX) {
-
-            @Override
-            public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
-                MetaData.Builder.toXContent(state, builder, formatParams);
-            }
-
-            @Override
-            public MetaData fromXContent(XContentParser parser) throws IOException {
-                return MetaData.Builder.fromXContent(parser);
-            }
-        };
-    }
-
-    /**
-     * Returns a StateFormat that can read and write {@link IndexMetaData}
-     */
-    static MetaDataStateFormat<IndexMetaData> indexStateFormat(XContentType format, final ToXContent.Params formatParams) {
-        return new MetaDataStateFormat<IndexMetaData>(format, INDEX_STATE_FILE_PREFIX) {
-
-            @Override
-            public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOException {
-                IndexMetaData.Builder.toXContent(state, builder, formatParams);            }
-
-            @Override
-            public IndexMetaData fromXContent(XContentParser parser) throws IOException {
-                return IndexMetaData.Builder.fromXContent(parser);
-            }
-        };
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java
index ed61aa2c1fd..e7447e301c4 100644
--- a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java
@@ -31,6 +31,7 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.NodeGatewayStartedShards;
 import org.elasticsearch.index.shard.ShardStateMetaData;
@@ -67,9 +68,13 @@ public abstract class PrimaryShardAllocator extends AbstractComponent {
         }
     };
 
-    public static final Setting<String> NODE_INITIAL_SHARDS_SETTING = new Setting<>("gateway.initial_shards", (settings) -> settings.get("gateway.local.initial_shards", "quorum"), INITIAL_SHARDS_PARSER, true, Setting.Scope.CLUSTER);
+    public static final Setting<String> NODE_INITIAL_SHARDS_SETTING =
+        new Setting<>("gateway.initial_shards", (settings) -> settings.get("gateway.local.initial_shards", "quorum"), INITIAL_SHARDS_PARSER,
+            Property.Dynamic, Property.NodeScope);
     @Deprecated
-    public static final Setting<String> INDEX_RECOVERY_INITIAL_SHARDS_SETTING = new Setting<>("index.recovery.initial_shards", (settings) -> NODE_INITIAL_SHARDS_SETTING.get(settings) , INITIAL_SHARDS_PARSER, true, Setting.Scope.INDEX);
+    public static final Setting<String> INDEX_RECOVERY_INITIAL_SHARDS_SETTING =
+        new Setting<>("index.recovery.initial_shards", (settings) -> NODE_INITIAL_SHARDS_SETTING.get(settings) , INITIAL_SHARDS_PARSER,
+            Property.Dynamic, Property.IndexScope);
 
     public PrimaryShardAllocator(Settings settings) {
         super(settings);
@@ -89,7 +94,7 @@ public abstract class PrimaryShardAllocator extends AbstractComponent {
                 continue;
             }
 
-            final IndexMetaData indexMetaData = metaData.index(shard.getIndexName());
+            final IndexMetaData indexMetaData = metaData.getIndexSafe(shard.index());
             // don't go wild here and create a new IndexSetting object for every shard this could cause a lot of garbage
             // on cluster restart if we allocate a boat load of shards
             if (shard.allocatedPostIndexCreate(indexMetaData) == false) {
@@ -113,7 +118,7 @@ public abstract class PrimaryShardAllocator extends AbstractComponent {
             final boolean enoughAllocationsFound;
 
             if (lastActiveAllocationIds.isEmpty()) {
-                assert Version.indexCreated(indexMetaData.getSettings()).before(Version.V_5_0_0) : "trying to allocated a primary with an empty allocation id set, but index is new";
+                assert Version.indexCreated(indexMetaData.getSettings()).before(Version.V_5_0_0_alpha1) : "trying to allocated a primary with an empty allocation id set, but index is new";
                 // when we load an old index (after upgrading cluster) or restore a snapshot of an old index
                 // fall back to old version-based allocation mode
                 // Note that once the shard has been active, lastActiveAllocationIds will be non-empty
@@ -123,7 +128,7 @@ public abstract class PrimaryShardAllocator extends AbstractComponent {
                 } else {
                     enoughAllocationsFound = isEnoughVersionBasedAllocationsFound(indexMetaData, nodeShardsResult);
                 }
-                logger.debug("[{}][{}]: version-based allocation for pre-{} index found {} allocations of {}", shard.index(), shard.id(), Version.V_5_0_0, nodeShardsResult.allocationsFound, shard);
+                logger.debug("[{}][{}]: version-based allocation for pre-{} index found {} allocations of {}", shard.index(), shard.id(), Version.V_5_0_0_alpha1, nodeShardsResult.allocationsFound, shard);
             } else {
                 assert lastActiveAllocationIds.isEmpty() == false;
                 // use allocation ids to select nodes
diff --git a/core/src/main/java/org/elasticsearch/gateway/PriorityComparator.java b/core/src/main/java/org/elasticsearch/gateway/PriorityComparator.java
index 04f438c70fe..1d24baf561a 100644
--- a/core/src/main/java/org/elasticsearch/gateway/PriorityComparator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/PriorityComparator.java
@@ -23,6 +23,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
 
 import java.util.Comparator;
 
@@ -42,8 +43,8 @@ public abstract class PriorityComparator implements Comparator<ShardRouting> {
         final String o2Index = o2.getIndexName();
         int cmp = 0;
         if (o1Index.equals(o2Index) == false) {
-            final Settings settingsO1 = getIndexSettings(o1Index);
-            final Settings settingsO2 = getIndexSettings(o2Index);
+            final Settings settingsO1 = getIndexSettings(o1.index());
+            final Settings settingsO2 = getIndexSettings(o2.index());
             cmp = Long.compare(priority(settingsO2), priority(settingsO1));
             if (cmp == 0) {
                 cmp = Long.compare(timeCreated(settingsO2), timeCreated(settingsO1));
@@ -63,7 +64,7 @@ public abstract class PriorityComparator implements Comparator<ShardRouting> {
         return settings.getAsLong(IndexMetaData.SETTING_CREATION_DATE, -1L);
     }
 
-    protected abstract Settings getIndexSettings(String index);
+    protected abstract Settings getIndexSettings(Index index);
 
     /**
      * Returns a PriorityComparator that uses the RoutingAllocation index metadata to access the index setting per index.
@@ -71,8 +72,8 @@ public abstract class PriorityComparator implements Comparator<ShardRouting> {
     public static PriorityComparator getAllocationComparator(final RoutingAllocation allocation) {
         return new PriorityComparator() {
             @Override
-            protected Settings getIndexSettings(String index) {
-                IndexMetaData indexMetaData = allocation.metaData().index(index);
+            protected Settings getIndexSettings(Index index) {
+                IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(index);
                 return indexMetaData.getSettings();
             }
         };
diff --git a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
index e2b6f0d27ed..74511639d47 100644
--- a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
@@ -74,7 +74,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
                 }
 
                 // if we are allocating a replica because of index creation, no need to go and find a copy, there isn't one...
-                IndexMetaData indexMetaData = metaData.index(shard.getIndexName());
+                IndexMetaData indexMetaData = metaData.getIndexSafe(shard.index());
                 if (shard.allocatedPostIndexCreate(indexMetaData) == false) {
                     continue;
                 }
@@ -104,6 +104,8 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
                             && matchingNodes.isNodeMatchBySyncID(nodeWithHighestMatch) == true) {
                         // we found a better match that has a full sync id match, the existing allocation is not fully synced
                         // so we found a better one, cancel this one
+                        logger.debug("cancelling allocation of replica on [{}], sync id match found on node [{}]",
+                                currentNode, nodeWithHighestMatch);
                         it.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.REALLOCATED_REPLICA,
                                 "existing allocation of replica to [" + currentNode + "] cancelled, sync id match found on node [" + nodeWithHighestMatch + "]",
                                 null, allocation.getCurrentNanoTime(), System.currentTimeMillis()));
@@ -127,7 +129,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
             }
 
             // if we are allocating a replica because of index creation, no need to go and find a copy, there isn't one...
-            IndexMetaData indexMetaData = metaData.index(shard.getIndexName());
+            IndexMetaData indexMetaData = metaData.getIndexSafe(shard.index());
             if (shard.allocatedPostIndexCreate(indexMetaData) == false) {
                 continue;
             }
diff --git a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
index fb174f4bd45..0fd1fd35809 100644
--- a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
+++ b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
@@ -29,10 +29,10 @@ import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
diff --git a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
index 03f8dc81703..7a090208818 100644
--- a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
+++ b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
@@ -29,11 +29,11 @@ import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -125,7 +125,7 @@ public class TransportNodesListGatewayStartedShards extends TransportNodesAction
             logger.trace("{} loading local shard state info", shardId);
             ShardStateMetaData shardStateMetaData = ShardStateMetaData.FORMAT.loadLatestState(logger, nodeEnv.availableShardPaths(request.shardId));
             if (shardStateMetaData != null) {
-                final IndexMetaData metaData = clusterService.state().metaData().index(shardId.getIndexName()); // it's a mystery why this is sometimes null
+                final IndexMetaData metaData = clusterService.state().metaData().index(shardId.getIndex()); // it's a mystery why this is sometimes null
                 if (metaData != null) {
                     ShardPath shardPath = null;
                     try {
diff --git a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
index 6c91df079b9..48af1c83965 100644
--- a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
+++ b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
@@ -20,42 +20,64 @@
 package org.elasticsearch.http;
 
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Scope;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.transport.PortsRange;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 
 import java.util.List;
+import java.util.function.Function;
 
 import static java.util.Collections.emptyList;
 import static org.elasticsearch.common.settings.Setting.listSetting;
 
 public final class HttpTransportSettings {
 
-    public static final Setting<Boolean> SETTING_CORS_ENABLED = Setting.boolSetting("http.cors.enabled", false, false, Scope.CLUSTER);
-    public static final Setting<String> SETTING_CORS_ALLOW_ORIGIN = new Setting<String>("http.cors.allow-origin", "", (value) -> value, false, Scope.CLUSTER);
-    public static final Setting<Integer> SETTING_CORS_MAX_AGE = Setting.intSetting("http.cors.max-age", 1728000, false, Scope.CLUSTER);
-    public static final Setting<String> SETTING_CORS_ALLOW_METHODS = new Setting<String>("http.cors.allow-methods", "OPTIONS, HEAD, GET, POST, PUT, DELETE", (value) -> value, false, Scope.CLUSTER);
-    public static final Setting<String> SETTING_CORS_ALLOW_HEADERS = new Setting<String>("http.cors.allow-headers", "X-Requested-With, Content-Type, Content-Length", (value) -> value, false, Scope.CLUSTER);
-    public static final Setting<Boolean> SETTING_CORS_ALLOW_CREDENTIALS = Setting.boolSetting("http.cors.allow-credentials", false, false, Scope.CLUSTER);
-    public static final Setting<Boolean> SETTING_PIPELINING = Setting.boolSetting("http.pipelining", true, false, Scope.CLUSTER);
-    public static final Setting<Integer> SETTING_PIPELINING_MAX_EVENTS = Setting.intSetting("http.pipelining.max_events", 10000, false, Scope.CLUSTER);
-    public static final Setting<Boolean> SETTING_HTTP_COMPRESSION = Setting.boolSetting("http.compression", false, false, Scope.CLUSTER);
-    public static final Setting<Integer> SETTING_HTTP_COMPRESSION_LEVEL = Setting.intSetting("http.compression_level", 6, false, Scope.CLUSTER);
-    public static final Setting<List<String>> SETTING_HTTP_HOST = listSetting("http.host", emptyList(), s -> s, false, Scope.CLUSTER);
-    public static final Setting<List<String>> SETTING_HTTP_PUBLISH_HOST = listSetting("http.publish_host", SETTING_HTTP_HOST, s -> s, false, Scope.CLUSTER);
-    public static final Setting<List<String>> SETTING_HTTP_BIND_HOST = listSetting("http.bind_host", SETTING_HTTP_HOST, s -> s, false, Scope.CLUSTER);
+    public static final Setting<Boolean> SETTING_CORS_ENABLED =
+        Setting.boolSetting("http.cors.enabled", false, Property.NodeScope);
+    public static final Setting<String> SETTING_CORS_ALLOW_ORIGIN =
+        new Setting<String>("http.cors.allow-origin", "", (value) -> value, Property.NodeScope);
+    public static final Setting<Integer> SETTING_CORS_MAX_AGE =
+        Setting.intSetting("http.cors.max-age", 1728000, Property.NodeScope);
+    public static final Setting<String> SETTING_CORS_ALLOW_METHODS =
+        new Setting<String>("http.cors.allow-methods", "OPTIONS, HEAD, GET, POST, PUT, DELETE", (value) -> value, Property.NodeScope);
+    public static final Setting<String> SETTING_CORS_ALLOW_HEADERS =
+        new Setting<String>("http.cors.allow-headers", "X-Requested-With, Content-Type, Content-Length", (value) -> value, Property.NodeScope);
+    public static final Setting<Boolean> SETTING_CORS_ALLOW_CREDENTIALS =
+        Setting.boolSetting("http.cors.allow-credentials", false, Property.NodeScope);
+    public static final Setting<Boolean> SETTING_PIPELINING =
+        Setting.boolSetting("http.pipelining", true, Property.NodeScope);
+    public static final Setting<Integer> SETTING_PIPELINING_MAX_EVENTS =
+        Setting.intSetting("http.pipelining.max_events", 10000, Property.NodeScope);
+    public static final Setting<Boolean> SETTING_HTTP_COMPRESSION =
+        Setting.boolSetting("http.compression", false, Property.NodeScope);
+    public static final Setting<Integer> SETTING_HTTP_COMPRESSION_LEVEL =
+        Setting.intSetting("http.compression_level", 6, Property.NodeScope);
+    public static final Setting<List<String>> SETTING_HTTP_HOST =
+        listSetting("http.host", emptyList(), Function.identity(), Property.NodeScope);
+    public static final Setting<List<String>> SETTING_HTTP_PUBLISH_HOST =
+        listSetting("http.publish_host", SETTING_HTTP_HOST, Function.identity(), Property.NodeScope);
+    public static final Setting<List<String>> SETTING_HTTP_BIND_HOST =
+        listSetting("http.bind_host", SETTING_HTTP_HOST, Function.identity(), Property.NodeScope);
 
-    public static final Setting<PortsRange> SETTING_HTTP_PORT = new Setting<PortsRange>("http.port", "9200-9300", PortsRange::new, false, Scope.CLUSTER);
-    public static final Setting<Integer> SETTING_HTTP_PUBLISH_PORT = Setting.intSetting("http.publish_port", -1, -1, false, Scope.CLUSTER);
-    public static final Setting<Boolean> SETTING_HTTP_DETAILED_ERRORS_ENABLED = Setting.boolSetting("http.detailed_errors.enabled", true, false, Scope.CLUSTER);
-    public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_CONTENT_LENGTH = Setting.byteSizeSetting("http.max_content_length", new ByteSizeValue(100, ByteSizeUnit.MB), false, Scope.CLUSTER) ;
-    public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_CHUNK_SIZE = Setting.byteSizeSetting("http.max_chunk_size", new ByteSizeValue(8, ByteSizeUnit.KB), false, Scope.CLUSTER) ;
-    public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_HEADER_SIZE = Setting.byteSizeSetting("http.max_header_size", new ByteSizeValue(8, ByteSizeUnit.KB), false, Scope.CLUSTER) ;
-    public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_INITIAL_LINE_LENGTH = Setting.byteSizeSetting("http.max_initial_line_length", new ByteSizeValue(4, ByteSizeUnit.KB), false, Scope.CLUSTER) ;
+    public static final Setting<PortsRange> SETTING_HTTP_PORT =
+        new Setting<PortsRange>("http.port", "9200-9300", PortsRange::new, Property.NodeScope);
+    public static final Setting<Integer> SETTING_HTTP_PUBLISH_PORT =
+        Setting.intSetting("http.publish_port", -1, -1, Property.NodeScope);
+    public static final Setting<Boolean> SETTING_HTTP_DETAILED_ERRORS_ENABLED =
+        Setting.boolSetting("http.detailed_errors.enabled", true, Property.NodeScope);
+    public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_CONTENT_LENGTH =
+        Setting.byteSizeSetting("http.max_content_length", new ByteSizeValue(100, ByteSizeUnit.MB), Property.NodeScope);
+    public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_CHUNK_SIZE =
+        Setting.byteSizeSetting("http.max_chunk_size", new ByteSizeValue(8, ByteSizeUnit.KB), Property.NodeScope);
+    public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_HEADER_SIZE =
+        Setting.byteSizeSetting("http.max_header_size", new ByteSizeValue(8, ByteSizeUnit.KB), Property.NodeScope);
+    public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_INITIAL_LINE_LENGTH =
+        Setting.byteSizeSetting("http.max_initial_line_length", new ByteSizeValue(4, ByteSizeUnit.KB), Property.NodeScope);
     // don't reset cookies by default, since I don't think we really need to
     // note, parsing cookies was fixed in netty 3.5.1 regarding stack allocation, but still, currently, we don't need cookies
-    public static final Setting<Boolean> SETTING_HTTP_RESET_COOKIES = Setting.boolSetting("http.reset_cookies", false, false, Scope.CLUSTER);
+    public static final Setting<Boolean> SETTING_HTTP_RESET_COOKIES =
+        Setting.boolSetting("http.reset_cookies", false, Property.NodeScope);
 
     private HttpTransportSettings() {
     }
diff --git a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java
index e64c6401f71..332380d9fb1 100644
--- a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java
+++ b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java
@@ -29,6 +29,7 @@ import org.elasticsearch.common.netty.OpenChannelsHandler;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
@@ -118,33 +119,32 @@ public class NettyHttpServerTransport extends AbstractLifecycleComponent<HttpSer
     }
 
     public static Setting<ByteSizeValue> SETTING_HTTP_NETTY_MAX_CUMULATION_BUFFER_CAPACITY =
-            Setting.byteSizeSetting("http.netty.max_cumulation_buffer_capacity", new ByteSizeValue(-1), false, Setting.Scope.CLUSTER);
+            Setting.byteSizeSetting("http.netty.max_cumulation_buffer_capacity", new ByteSizeValue(-1),
+                Property.NodeScope);
     public static Setting<Integer> SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS =
-            Setting.intSetting("http.netty.max_composite_buffer_components", -1, false, Setting.Scope.CLUSTER);
+            Setting.intSetting("http.netty.max_composite_buffer_components", -1, Property.NodeScope);
 
     public static final Setting<Integer> SETTING_HTTP_WORKER_COUNT = new Setting<>("http.netty.worker_count",
             (s) -> Integer.toString(EsExecutors.boundedNumberOfProcessors(s) * 2),
-            (s) -> Setting.parseInt(s, 1, "http.netty.worker_count"), false, Setting.Scope.CLUSTER);
+            (s) -> Setting.parseInt(s, 1, "http.netty.worker_count"), Property.NodeScope);
 
-    public static final Setting<Boolean> SETTING_HTTP_TCP_NO_DELAY = boolSetting("http.tcp_no_delay", NetworkService.TcpSettings
-            .TCP_NO_DELAY, false,
-            Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> SETTING_HTTP_TCP_KEEP_ALIVE = boolSetting("http.tcp.keep_alive", NetworkService.TcpSettings
-            .TCP_KEEP_ALIVE, false,
-            Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> SETTING_HTTP_TCP_BLOCKING_SERVER = boolSetting("http.tcp.blocking_server", NetworkService
-            .TcpSettings.TCP_BLOCKING_SERVER,
-            false, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> SETTING_HTTP_TCP_REUSE_ADDRESS = boolSetting("http.tcp.reuse_address", NetworkService
-            .TcpSettings.TCP_REUSE_ADDRESS,
-            false, Setting.Scope.CLUSTER);
+    public static final Setting<Boolean> SETTING_HTTP_TCP_NO_DELAY =
+        boolSetting("http.tcp_no_delay", NetworkService.TcpSettings.TCP_NO_DELAY, Property.NodeScope);
+    public static final Setting<Boolean> SETTING_HTTP_TCP_KEEP_ALIVE =
+        boolSetting("http.tcp.keep_alive", NetworkService.TcpSettings.TCP_KEEP_ALIVE, Property.NodeScope);
+    public static final Setting<Boolean> SETTING_HTTP_TCP_BLOCKING_SERVER =
+        boolSetting("http.tcp.blocking_server", NetworkService.TcpSettings.TCP_BLOCKING_SERVER, Property.NodeScope);
+    public static final Setting<Boolean> SETTING_HTTP_TCP_REUSE_ADDRESS =
+        boolSetting("http.tcp.reuse_address", NetworkService.TcpSettings.TCP_REUSE_ADDRESS, Property.NodeScope);
 
-    public static final Setting<ByteSizeValue> SETTING_HTTP_TCP_SEND_BUFFER_SIZE = Setting.byteSizeSetting("http.tcp.send_buffer_size",
-            NetworkService.TcpSettings.TCP_SEND_BUFFER_SIZE, false, Setting.Scope.CLUSTER);
-    public static final Setting<ByteSizeValue> SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE = Setting.byteSizeSetting("http.tcp" +
-            ".receive_buffer_size", NetworkService.TcpSettings.TCP_RECEIVE_BUFFER_SIZE, false, Setting.Scope.CLUSTER);
-    public static final Setting<ByteSizeValue> SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE = Setting.byteSizeSetting(
-            "transport.netty.receive_predictor_size",
+    public static final Setting<ByteSizeValue> SETTING_HTTP_TCP_SEND_BUFFER_SIZE =
+        Setting.byteSizeSetting("http.tcp.send_buffer_size", NetworkService.TcpSettings.TCP_SEND_BUFFER_SIZE,
+            Property.NodeScope);
+    public static final Setting<ByteSizeValue> SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE =
+        Setting.byteSizeSetting("http.tcp.receive_buffer_size", NetworkService.TcpSettings.TCP_RECEIVE_BUFFER_SIZE,
+            Property.NodeScope);
+    public static final Setting<ByteSizeValue> SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE =
+        Setting.byteSizeSetting("transport.netty.receive_predictor_size",
             settings -> {
                 long defaultReceiverPredictor = 512 * 1024;
                 if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes() > 0) {
@@ -154,13 +154,11 @@ public class NettyHttpServerTransport extends AbstractLifecycleComponent<HttpSer
                     defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024));
                 }
                 return new ByteSizeValue(defaultReceiverPredictor).toString();
-            }, false, Setting.Scope.CLUSTER);
-    public static final Setting<ByteSizeValue> SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MIN = byteSizeSetting("http.netty" +
-            ".receive_predictor_min",
-            SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE, false, Setting.Scope.CLUSTER);
-    public static final Setting<ByteSizeValue> SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MAX = byteSizeSetting("http.netty" +
-            ".receive_predictor_max",
-            SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE, false, Setting.Scope.CLUSTER);
+            }, Property.NodeScope);
+    public static final Setting<ByteSizeValue> SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MIN =
+        byteSizeSetting("http.netty.receive_predictor_min", SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope);
+    public static final Setting<ByteSizeValue> SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MAX =
+        byteSizeSetting("http.netty.receive_predictor_max", SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope);
 
 
     protected final NetworkService networkService;
@@ -262,7 +260,7 @@ public class NettyHttpServerTransport extends AbstractLifecycleComponent<HttpSer
 
         // validate max content length
         if (maxContentLength.bytes() > Integer.MAX_VALUE) {
-            logger.warn("maxContentLength[" + maxContentLength + "] set to high value, resetting it to [100mb]");
+            logger.warn("maxContentLength[{}] set to high value, resetting it to [100mb]", maxContentLength);
             maxContentLength = new ByteSizeValue(100, ByteSizeUnit.MB);
         }
         this.maxContentLength = maxContentLength;
diff --git a/core/src/main/java/org/elasticsearch/index/Index.java b/core/src/main/java/org/elasticsearch/index/Index.java
index 80bf3c31b44..3ffe13e38b1 100644
--- a/core/src/main/java/org/elasticsearch/index/Index.java
+++ b/core/src/main/java/org/elasticsearch/index/Index.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index;
 
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -30,7 +31,7 @@ import java.io.IOException;
  */
 public class Index implements Writeable<Index> {
 
-    private final static Index PROTO = new Index("", "");
+    public static final Index[] EMPTY_ARRAY = new Index[0];
 
     private final String name;
     private final String uuid;
@@ -40,6 +41,12 @@ public class Index implements Writeable<Index> {
         this.uuid = uuid.intern();
     }
 
+    public Index(StreamInput in) throws IOException {
+        this.name = in.readString();
+        this.uuid = in.readString();
+    }
+
+
     public String getName() {
         return this.name;
     }
@@ -50,7 +57,14 @@ public class Index implements Writeable<Index> {
 
     @Override
     public String toString() {
-        return "[" + name + "]";
+        /*
+         * If we have a uuid we put it in the toString so it'll show up in logs which is useful as more and more things use the uuid rather
+         * than the name as the lookup key for the index.
+         */
+        if (ClusterState.UNKNOWN_UUID.equals(uuid)) {
+            return "[" + name + "]";
+        }
+        return "[" + name + "/" + uuid + "]";
     }
 
     @Override
@@ -72,13 +86,9 @@ public class Index implements Writeable<Index> {
         return result;
     }
 
-    public static Index readIndex(StreamInput in) throws IOException {
-        return PROTO.readFrom(in);
-    }
-
     @Override
     public Index readFrom(StreamInput in) throws IOException {
-        return new Index(in.readString(), in.readString());
+        return new Index(in);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/IndexModule.java b/core/src/main/java/org/elasticsearch/index/IndexModule.java
index eabc0951e7f..48230e6ec1e 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexModule.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexModule.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
@@ -65,13 +66,17 @@ import java.util.function.Function;
  */
 public final class IndexModule {
 
-    public static final Setting<String> INDEX_STORE_TYPE_SETTING = new Setting<>("index.store.type", "", Function.identity(), false, Setting.Scope.INDEX);
+    public static final Setting<String> INDEX_STORE_TYPE_SETTING =
+        new Setting<>("index.store.type", "", Function.identity(), Property.IndexScope, Property.NodeScope);
     public static final String SIMILARITY_SETTINGS_PREFIX = "index.similarity";
     public static final String INDEX_QUERY_CACHE = "index";
     public static final String NONE_QUERY_CACHE = "none";
-    public static final Setting<String> INDEX_QUERY_CACHE_TYPE_SETTING = new Setting<>("index.queries.cache.type", INDEX_QUERY_CACHE, Function.identity(), false, Setting.Scope.INDEX);
+    public static final Setting<String> INDEX_QUERY_CACHE_TYPE_SETTING =
+        new Setting<>("index.queries.cache.type", INDEX_QUERY_CACHE, Function.identity(), Property.IndexScope);
+
     // for test purposes only
-    public static final Setting<Boolean> INDEX_QUERY_CACHE_EVERYTHING_SETTING = Setting.boolSetting("index.queries.cache.everything", false, false, Setting.Scope.INDEX);
+    public static final Setting<Boolean> INDEX_QUERY_CACHE_EVERYTHING_SETTING =
+        Setting.boolSetting("index.queries.cache.everything", false, Property.IndexScope);
     private final IndexSettings indexSettings;
     private final IndexStoreConfig indexStoreConfig;
     private final AnalysisRegistry analysisRegistry;
@@ -83,7 +88,7 @@ public final class IndexModule {
     private final Map<String, BiFunction<String, Settings, SimilarityProvider>> similarities = new HashMap<>();
     private final Map<String, BiFunction<IndexSettings, IndexStoreConfig, IndexStore>> storeTypes = new HashMap<>();
     private final Map<String, BiFunction<IndexSettings, IndicesQueryCache, QueryCache>> queryCaches = new HashMap<>();
-
+    private final SetOnce<String> forceQueryCacheType = new SetOnce<>();
 
     public IndexModule(IndexSettings indexSettings, IndexStoreConfig indexStoreConfig, AnalysisRegistry analysisRegistry) {
         this.indexStoreConfig = indexStoreConfig;
@@ -261,11 +266,23 @@ public final class IndexModule {
         }
         indexSettings.getScopedSettings().addSettingsUpdateConsumer(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING, store::setType);
         indexSettings.getScopedSettings().addSettingsUpdateConsumer(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING, store::setMaxRate);
-        final String queryCacheType = indexSettings.getValue(INDEX_QUERY_CACHE_TYPE_SETTING);
+        final String queryCacheType = forceQueryCacheType.get() != null ? forceQueryCacheType.get() : indexSettings.getValue(INDEX_QUERY_CACHE_TYPE_SETTING);
         final BiFunction<IndexSettings, IndicesQueryCache, QueryCache> queryCacheProvider = queryCaches.get(queryCacheType);
         final QueryCache queryCache = queryCacheProvider.apply(indexSettings, indicesQueryCache);
         return new IndexService(indexSettings, environment, new SimilarityService(indexSettings, similarities), shardStoreDeleter, analysisRegistry, engineFactory.get(),
                 servicesProvider, queryCache, store, eventListener, searcherWrapperFactory, mapperRegistry, indicesFieldDataCache, listeners);
     }
 
+    /**
+     * Forces a certain query cache type. If this is set
+     * the given cache type is overriding the default as well as the type
+     * set on the index level.
+     * NOTE: this can only be set once
+     *
+     * @see #INDEX_QUERY_CACHE_TYPE_SETTING
+     */
+    public void forceQueryCacheType(String type) {
+        this.forceQueryCacheType.set(type);
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java
index 6b4a1851ab5..815f257a45d 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexService.java
@@ -19,18 +19,6 @@
 
 package org.elasticsearch.index;
 
-import java.io.Closeable;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Query;
@@ -44,6 +32,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -57,10 +46,10 @@ import org.elasticsearch.index.cache.query.QueryCache;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineClosedException;
 import org.elasticsearch.index.engine.EngineFactory;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.percolator.PercolatorQueryCache;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexEventListener;
@@ -81,6 +70,18 @@ import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.unmodifiableMap;
 import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder;
@@ -113,6 +114,8 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
     private volatile AsyncRefreshTask refreshTask;
     private volatile AsyncTranslogFSync fsyncTask;
     private final SearchSlowLog searchSlowLog;
+    private final ThreadPool threadPool;
+    private final BigArrays bigArrays;
 
     public IndexService(IndexSettings indexSettings, NodeEnvironment nodeEnv,
                         SimilarityService similarityService,
@@ -131,17 +134,24 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
         this.indexSettings = indexSettings;
         this.analysisService = registry.build(indexSettings);
         this.similarityService = similarityService;
-        this.mapperService = new MapperService(indexSettings, analysisService, similarityService, mapperRegistry, IndexService.this::newQueryShardContext);
-        this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, nodeServicesProvider.getCircuitBreakerService(), mapperService);
+        this.mapperService = new MapperService(indexSettings, analysisService, similarityService, mapperRegistry,
+            IndexService.this::newQueryShardContext);
+        this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache,
+            nodeServicesProvider.getCircuitBreakerService(), mapperService);
         this.shardStoreDeleter = shardStoreDeleter;
+        this.bigArrays = nodeServicesProvider.getBigArrays();
+        this.threadPool = nodeServicesProvider.getThreadPool();
         this.eventListener = eventListener;
         this.nodeEnv = nodeEnv;
         this.nodeServicesProvider = nodeServicesProvider;
         this.indexStore = indexStore;
         indexFieldData.setListener(new FieldDataCacheListener(this));
         this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
-        this.warmer = new IndexWarmer(indexSettings.getSettings(), nodeServicesProvider.getThreadPool(), bitsetFilterCache.createListener(nodeServicesProvider.getThreadPool()));
-        this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache);
+        PercolatorQueryCache percolatorQueryCache = new PercolatorQueryCache(indexSettings, IndexService.this::newQueryShardContext);
+        this.warmer = new IndexWarmer(indexSettings.getSettings(), threadPool,
+            bitsetFilterCache.createListener(threadPool),
+            percolatorQueryCache.createListener(threadPool));
+        this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache, percolatorQueryCache);
         this.engineFactory = engineFactory;
         // initialize this last -- otherwise if the wrapper requires any other member to be non-null we fail with an NPE
         this.searcherWrapper = wrapperFactory.newWrapper(this);
@@ -230,7 +240,8 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
                     }
                 }
             } finally {
-                IOUtils.close(bitsetFilterCache, indexCache, mapperService, indexFieldData, analysisService, refreshTask, fsyncTask);
+                IOUtils.close(bitsetFilterCache, indexCache, mapperService, indexFieldData, analysisService, refreshTask, fsyncTask,
+                    cache().getPercolatorQueryCache());
             }
         }
     }
@@ -300,7 +311,9 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
                     }
                     dataPathToShardCount.put(dataPath, curCount + 1);
                 }
-                path = ShardPath.selectNewPathForShard(nodeEnv, shardId, this.indexSettings, routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE ? getAvgShardSizeInBytes() : routing.getExpectedShardSize(),
+                path = ShardPath.selectNewPathForShard(nodeEnv, shardId, this.indexSettings,
+                    routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE
+                        ? getAvgShardSizeInBytes() : routing.getExpectedShardSize(),
                     dataPathToShardCount);
                 logger.debug("{} creating using a new path [{}]", shardId, path);
             } else {
@@ -315,17 +328,22 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
             // if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the primary.
             final boolean canDeleteShardContent = IndexMetaData.isOnSharedFilesystem(indexSettings) == false ||
                 (primary && IndexMetaData.isOnSharedFilesystem(indexSettings));
-            final Engine.Warmer engineWarmer = (searcher, toLevel) -> {
+            final Engine.Warmer engineWarmer = (searcher) -> {
                 IndexShard shard =  getShardOrNull(shardId.getId());
                 if (shard != null) {
-                    warmer.warm(searcher, shard, IndexService.this.indexSettings, toLevel);
+                    warmer.warm(searcher, shard, IndexService.this.indexSettings);
                 }
             };
-            store = new Store(shardId, this.indexSettings, indexStore.newDirectoryService(path), lock, new StoreCloseListener(shardId, canDeleteShardContent, () -> eventListener.onStoreClosed(shardId)));
+            store = new Store(shardId, this.indexSettings, indexStore.newDirectoryService(path), lock,
+                new StoreCloseListener(shardId, canDeleteShardContent, () -> eventListener.onStoreClosed(shardId)));
             if (useShadowEngine(primary, indexSettings)) {
-                indexShard = new ShadowIndexShard(shardId, this.indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldData, engineFactory, eventListener, searcherWrapper, nodeServicesProvider, searchSlowLog, engineWarmer); // no indexing listeners - shadow  engines don't index
+                indexShard = new ShadowIndexShard(shardId, this.indexSettings, path, store, indexCache, mapperService, similarityService,
+                    indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, searchSlowLog, engineWarmer);
+                // no indexing listeners - shadow  engines don't index
             } else {
-                indexShard = new IndexShard(shardId, this.indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldData, engineFactory, eventListener, searcherWrapper, nodeServicesProvider, searchSlowLog, engineWarmer, listeners);
+                indexShard = new IndexShard(shardId, this.indexSettings, path, store, indexCache, mapperService, similarityService,
+                    indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, searchSlowLog, engineWarmer,
+                    listeners);
             }
             eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
             eventListener.afterIndexShardCreated(indexShard);
@@ -370,7 +388,8 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
                 // and close the shard so no operations are allowed to it
                 if (indexShard != null) {
                     try {
-                        final boolean flushEngine = deleted.get() == false && closed.get(); // only flush we are we closed (closed index or shutdown) and if we are not deleted
+                        // only flush we are we closed (closed index or shutdown) and if we are not deleted
+                        final boolean flushEngine = deleted.get() == false && closed.get();
                         indexShard.close(reason, flushEngine);
                     } catch (Throwable e) {
                         logger.debug("[{}] failed to close index shard", e, shardId);
@@ -417,14 +436,23 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
     }
 
     /**
-     * Creates a new QueryShardContext. The context has not types set yet, if types are required set them via {@link QueryShardContext#setTypes(String...)}
+     * Creates a new QueryShardContext. The context has not types set yet, if types are required set them via
+     * {@link QueryShardContext#setTypes(String...)}
      */
     public QueryShardContext newQueryShardContext() {
-        return new QueryShardContext(indexSettings, indexCache.bitsetFilterCache(), indexFieldData, mapperService(), similarityService(), nodeServicesProvider.getScriptService(), nodeServicesProvider.getIndicesQueriesRegistry());
+        return new QueryShardContext(
+                indexSettings, indexCache.bitsetFilterCache(), indexFieldData, mapperService(),
+                similarityService(), nodeServicesProvider.getScriptService(), nodeServicesProvider.getIndicesQueriesRegistry(),
+                indexCache.getPercolatorQueryCache()
+        );
     }
 
-    ThreadPool getThreadPool() {
-        return nodeServicesProvider.getThreadPool();
+    public ThreadPool getThreadPool() {
+        return threadPool;
+    }
+
+    public BigArrays getBigArrays() {
+        return bigArrays;
     }
 
     public SearchSlowLog getSearchSlowLog() {
@@ -496,21 +524,21 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
         }
 
         @Override
-        public void onCache(ShardId shardId, String fieldName, FieldDataType fieldDataType, Accountable ramUsage) {
+        public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) {
             if (shardId != null) {
                 final IndexShard shard = indexService.getShardOrNull(shardId.id());
                 if (shard != null) {
-                    shard.fieldData().onCache(shardId, fieldName, fieldDataType, ramUsage);
+                    shard.fieldData().onCache(shardId, fieldName, ramUsage);
                 }
             }
         }
 
         @Override
-        public void onRemoval(ShardId shardId, String fieldName, FieldDataType fieldDataType, boolean wasEvicted, long sizeInBytes) {
+        public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) {
             if (shardId != null) {
                 final IndexShard shard = indexService.getShardOrNull(shardId.id());
                 if (shard != null) {
-                    shard.fieldData().onRemoval(shardId, fieldName, fieldDataType, wasEvicted, sizeInBytes);
+                    shard.fieldData().onRemoval(shardId, fieldName, wasEvicted, sizeInBytes);
                 }
             }
         }
@@ -541,7 +569,8 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
                 AliasMetaData alias = aliases.get(aliasName);
                 if (alias == null) {
                     // This shouldn't happen unless alias disappeared after filteringAliases was called.
-                    throw new InvalidAliasNameException(indexSettings.getIndex(), aliasNames[0], "Unknown alias name was passed to alias Filter");
+                    throw new InvalidAliasNameException(indexSettings.getIndex(), aliasNames[0],
+                        "Unknown alias name was passed to alias Filter");
                 }
                 Query parsedFilter = parse(alias, context);
                 if (parsedFilter != null) {
@@ -592,6 +621,11 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
                 rescheduleFsyncTask(durability);
             }
         }
+
+        // update primary terms
+        for (final IndexShard shard : this.shards.values()) {
+            shard.updatePrimaryTerm(metadata.primaryTerm(shard.shardId().id()));
+        }
     }
 
     private void rescheduleFsyncTask(Translog.Durability durability) {
@@ -717,7 +751,8 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
             } catch (Exception ex) {
                 if (lastThrownException == null || sameException(lastThrownException, ex) == false) {
                     // prevent the annoying fact of logging the same stuff all the time with an interval of 1 sec will spam all your logs
-                    indexService.logger.warn("failed to run task {} - suppressing re-occurring exceptions unless the exception changes", ex, toString());
+                    indexService.logger.warn("failed to run task {} - suppressing re-occurring exceptions unless the exception changes",
+                        ex, toString());
                     lastThrownException = ex;
                 }
             } finally {
@@ -750,7 +785,7 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
         }
 
         @Override
-        public void close() {
+        public synchronized void close() {
             if (closed.compareAndSet(false, true)) {
                 FutureUtils.cancel(scheduledFuture);
                 scheduledFuture = null;
diff --git a/core/src/main/java/org/elasticsearch/index/IndexSettings.java b/core/src/main/java/org/elasticsearch/index/IndexSettings.java
index b17b8ab7edf..b996e70b1e5 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexSettings.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -36,7 +37,6 @@ import org.elasticsearch.index.translog.Translog;
 
 import java.util.Locale;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Predicate;
@@ -50,15 +50,26 @@ import java.util.function.Predicate;
  */
 public final class IndexSettings {
 
-    public static final Setting<String> DEFAULT_FIELD_SETTING = new Setting<>("index.query.default_field", AllFieldMapper.NAME, Function.identity(), false, Setting.Scope.INDEX);
-    public static final Setting<Boolean> QUERY_STRING_LENIENT_SETTING = Setting.boolSetting("index.query_string.lenient", false, false, Setting.Scope.INDEX);
-    public static final Setting<Boolean> QUERY_STRING_ANALYZE_WILDCARD = Setting.boolSetting("indices.query.query_string.analyze_wildcard", false, false, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> QUERY_STRING_ALLOW_LEADING_WILDCARD = Setting.boolSetting("indices.query.query_string.allowLeadingWildcard", true, false, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> ALLOW_UNMAPPED = Setting.boolSetting("index.query.parse.allow_unmapped_fields", true, false, Setting.Scope.INDEX);
-    public static final Setting<TimeValue> INDEX_TRANSLOG_SYNC_INTERVAL_SETTING = Setting.timeSetting("index.translog.sync_interval", TimeValue.timeValueSeconds(5), TimeValue.timeValueMillis(100), false, Setting.Scope.INDEX);
-    public static final Setting<Translog.Durability> INDEX_TRANSLOG_DURABILITY_SETTING = new Setting<>("index.translog.durability", Translog.Durability.REQUEST.name(), (value) -> Translog.Durability.valueOf(value.toUpperCase(Locale.ROOT)), true, Setting.Scope.INDEX);
-    public static final Setting<Boolean> INDEX_WARMER_ENABLED_SETTING = Setting.boolSetting("index.warmer.enabled", true, true, Setting.Scope.INDEX);
-    public static final Setting<Boolean> INDEX_TTL_DISABLE_PURGE_SETTING = Setting.boolSetting("index.ttl.disable_purge", false, true, Setting.Scope.INDEX);
+    public static final Setting<String> DEFAULT_FIELD_SETTING =
+        new Setting<>("index.query.default_field", AllFieldMapper.NAME, Function.identity(), Property.IndexScope);
+    public static final Setting<Boolean> QUERY_STRING_LENIENT_SETTING =
+        Setting.boolSetting("index.query_string.lenient", false, Property.IndexScope);
+    public static final Setting<Boolean> QUERY_STRING_ANALYZE_WILDCARD =
+        Setting.boolSetting("indices.query.query_string.analyze_wildcard", false, Property.NodeScope);
+    public static final Setting<Boolean> QUERY_STRING_ALLOW_LEADING_WILDCARD =
+        Setting.boolSetting("indices.query.query_string.allowLeadingWildcard", true, Property.NodeScope);
+    public static final Setting<Boolean> ALLOW_UNMAPPED =
+        Setting.boolSetting("index.query.parse.allow_unmapped_fields", true, Property.IndexScope);
+    public static final Setting<TimeValue> INDEX_TRANSLOG_SYNC_INTERVAL_SETTING =
+        Setting.timeSetting("index.translog.sync_interval", TimeValue.timeValueSeconds(5), TimeValue.timeValueMillis(100),
+            Property.IndexScope);
+    public static final Setting<Translog.Durability> INDEX_TRANSLOG_DURABILITY_SETTING =
+        new Setting<>("index.translog.durability", Translog.Durability.REQUEST.name(),
+            (value) -> Translog.Durability.valueOf(value.toUpperCase(Locale.ROOT)), Property.Dynamic, Property.IndexScope);
+    public static final Setting<Boolean> INDEX_WARMER_ENABLED_SETTING =
+        Setting.boolSetting("index.warmer.enabled", true, Property.Dynamic, Property.IndexScope);
+    public static final Setting<Boolean> INDEX_TTL_DISABLE_PURGE_SETTING =
+        Setting.boolSetting("index.ttl.disable_purge", false, Property.Dynamic, Property.IndexScope);
     public static final Setting<String> INDEX_CHECK_ON_STARTUP = new Setting<>("index.shard.check_on_startup", "false", (s) -> {
         switch(s) {
             case "false":
@@ -69,7 +80,7 @@ public final class IndexSettings {
             default:
                 throw new IllegalArgumentException("unknown value for [index.shard.check_on_startup] must be one of [true, false, fix, checksum] but was: " + s);
         }
-    }, false, Setting.Scope.INDEX);
+    }, Property.IndexScope);
 
     /**
      * Index setting describing the maximum value of from + size on a query.
@@ -79,10 +90,15 @@ public final class IndexSettings {
      * safely. 1,000,000 is probably way to high for any cluster to set
      * safely.
      */
-    public static final Setting<Integer> MAX_RESULT_WINDOW_SETTING = Setting.intSetting("index.max_result_window", 10000, 1, true, Setting.Scope.INDEX);
+    public static final Setting<Integer> MAX_RESULT_WINDOW_SETTING =
+        Setting.intSetting("index.max_result_window", 10000, 1, Property.Dynamic, Property.IndexScope);
     public static final TimeValue DEFAULT_REFRESH_INTERVAL = new TimeValue(1, TimeUnit.SECONDS);
-    public static final Setting<TimeValue> INDEX_REFRESH_INTERVAL_SETTING = Setting.timeSetting("index.refresh_interval", DEFAULT_REFRESH_INTERVAL, new TimeValue(-1, TimeUnit.MILLISECONDS), true, Setting.Scope.INDEX);
-    public static final Setting<ByteSizeValue> INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING = Setting.byteSizeSetting("index.translog.flush_threshold_size", new ByteSizeValue(512, ByteSizeUnit.MB), true, Setting.Scope.INDEX);
+    public static final Setting<TimeValue> INDEX_REFRESH_INTERVAL_SETTING =
+        Setting.timeSetting("index.refresh_interval", DEFAULT_REFRESH_INTERVAL, new TimeValue(-1, TimeUnit.MILLISECONDS),
+            Property.Dynamic, Property.IndexScope);
+    public static final Setting<ByteSizeValue> INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING =
+        Setting.byteSizeSetting("index.translog.flush_threshold_size", new ByteSizeValue(512, ByteSizeUnit.MB), Property.Dynamic,
+            Property.IndexScope);
 
 
     /**
@@ -90,7 +106,9 @@ public final class IndexSettings {
      * This setting is realtime updateable
      */
     public static final TimeValue DEFAULT_GC_DELETES = TimeValue.timeValueSeconds(60);
-    public static final Setting<TimeValue> INDEX_GC_DELETES_SETTING = Setting.timeSetting("index.gc_deletes", DEFAULT_GC_DELETES, new TimeValue(-1, TimeUnit.MILLISECONDS), true, Setting.Scope.INDEX);
+    public static final Setting<TimeValue> INDEX_GC_DELETES_SETTING =
+        Setting.timeSetting("index.gc_deletes", DEFAULT_GC_DELETES, new TimeValue(-1, TimeUnit.MILLISECONDS), Property.Dynamic,
+            Property.IndexScope);
 
     private final Index index;
     private final Version version;
diff --git a/core/src/main/java/org/elasticsearch/index/IndexWarmer.java b/core/src/main/java/org/elasticsearch/index/IndexWarmer.java
index 9fabc8efc40..499031af970 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexWarmer.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexWarmer.java
@@ -19,18 +19,11 @@
 
 package org.elasticsearch.index;
 
-import com.carrotsearch.hppc.ObjectHashSet;
-import com.carrotsearch.hppc.ObjectSet;
-import com.carrotsearch.hppc.cursors.ObjectCursor;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.DirectoryReader;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.DocumentMapper;
@@ -54,16 +47,12 @@ import java.util.concurrent.TimeUnit;
  */
 public final class IndexWarmer extends AbstractComponent {
 
-    public static final Setting<MappedFieldType.Loading> INDEX_NORMS_LOADING_SETTING = new Setting<>("index.norms.loading",
-        MappedFieldType.Loading.LAZY.toString(), (s) -> MappedFieldType.Loading.parse(s, MappedFieldType.Loading.LAZY),
-        false, Setting.Scope.INDEX);
     private final List<Listener> listeners;
 
     IndexWarmer(Settings settings, ThreadPool threadPool, Listener... listeners) {
         super(settings);
         ArrayList<Listener> list = new ArrayList<>();
         final Executor executor = threadPool.executor(ThreadPool.Names.WARMER);
-        list.add(new NormsWarmer(executor));
         list.add(new FieldDataWarmer(executor));
         for (Listener listener : listeners) {
             list.add(listener);
@@ -71,7 +60,7 @@ public final class IndexWarmer extends AbstractComponent {
         this.listeners = Collections.unmodifiableList(list);
     }
 
-    void warm(Engine.Searcher searcher, IndexShard shard, IndexSettings settings, boolean isTopReader) {
+    void warm(Engine.Searcher searcher, IndexShard shard, IndexSettings settings) {
         if (shard.state() == IndexShardState.CLOSED) {
             return;
         }
@@ -79,22 +68,14 @@ public final class IndexWarmer extends AbstractComponent {
             return;
         }
         if (logger.isTraceEnabled()) {
-            if (isTopReader) {
-                logger.trace("{} top warming [{}]", shard.shardId(), searcher.reader());
-            } else {
-                logger.trace("{} warming [{}]", shard.shardId(), searcher.reader());
-            }
+            logger.trace("{} top warming [{}]", shard.shardId(), searcher.reader());
         }
         shard.warmerService().onPreWarm();
         long time = System.nanoTime();
         final List<TerminationHandle> terminationHandles = new ArrayList<>();
         // get a handle on pending tasks
         for (final Listener listener : listeners) {
-            if (isTopReader) {
-                terminationHandles.add(listener.warmTopReader(shard, searcher));
-            } else {
-                terminationHandles.add(listener.warmNewReaders(shard, searcher));
-            }
+            terminationHandles.add(listener.warmReader(shard, searcher));
         }
         // wait for termination
         for (TerminationHandle terminationHandle : terminationHandles) {
@@ -102,22 +83,14 @@ public final class IndexWarmer extends AbstractComponent {
                 terminationHandle.awaitTermination();
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
-                if (isTopReader) {
-                    logger.warn("top warming has been interrupted", e);
-                } else {
-                    logger.warn("warming has been interrupted", e);
-                }
+                logger.warn("top warming has been interrupted", e);
                 break;
             }
         }
         long took = System.nanoTime() - time;
         shard.warmerService().onPostWarm(took);
         if (shard.warmerService().logger().isTraceEnabled()) {
-            if (isTopReader) {
-                shard.warmerService().logger().trace("top warming took [{}]", new TimeValue(took, TimeUnit.NANOSECONDS));
-            } else {
-                shard.warmerService().logger().trace("warming took [{}]", new TimeValue(took, TimeUnit.NANOSECONDS));
-            }
+            shard.warmerService().logger().trace("top warming took [{}]", new TimeValue(took, TimeUnit.NANOSECONDS));
         }
     }
 
@@ -132,67 +105,7 @@ public final class IndexWarmer extends AbstractComponent {
     public interface Listener {
         /** Queue tasks to warm-up the given segments and return handles that allow to wait for termination of the
          *  execution of those tasks. */
-        TerminationHandle warmNewReaders(IndexShard indexShard, Engine.Searcher searcher);
-
-        TerminationHandle warmTopReader(IndexShard indexShard, Engine.Searcher searcher);
-    }
-
-    private static class NormsWarmer implements IndexWarmer.Listener {
-        private final Executor executor;
-        public NormsWarmer(Executor executor) {
-            this.executor = executor;
-        }
-        @Override
-        public TerminationHandle warmNewReaders(final IndexShard indexShard, final Engine.Searcher searcher) {
-            final MappedFieldType.Loading defaultLoading = indexShard.indexSettings().getValue(INDEX_NORMS_LOADING_SETTING);
-            final MapperService mapperService = indexShard.mapperService();
-            final ObjectSet<String> warmUp = new ObjectHashSet<>();
-            for (DocumentMapper docMapper : mapperService.docMappers(false)) {
-                for (FieldMapper fieldMapper : docMapper.mappers()) {
-                    final String indexName = fieldMapper.fieldType().name();
-                    MappedFieldType.Loading normsLoading = fieldMapper.fieldType().normsLoading();
-                    if (normsLoading == null) {
-                        normsLoading = defaultLoading;
-                    }
-                    if (fieldMapper.fieldType().indexOptions() != IndexOptions.NONE && !fieldMapper.fieldType().omitNorms()
-                        && normsLoading == MappedFieldType.Loading.EAGER) {
-                        warmUp.add(indexName);
-                    }
-                }
-            }
-
-            final CountDownLatch latch = new CountDownLatch(1);
-            // Norms loading may be I/O intensive but is not CPU intensive, so we execute it in a single task
-            executor.execute(() -> {
-                try {
-                    for (ObjectCursor<String> stringObjectCursor : warmUp) {
-                        final String indexName = stringObjectCursor.value;
-                        final long start = System.nanoTime();
-                        for (final LeafReaderContext ctx : searcher.reader().leaves()) {
-                            final NumericDocValues values = ctx.reader().getNormValues(indexName);
-                            if (values != null) {
-                                values.get(0);
-                            }
-                        }
-                        if (indexShard.warmerService().logger().isTraceEnabled()) {
-                            indexShard.warmerService().logger().trace("warmed norms for [{}], took [{}]", indexName,
-                                TimeValue.timeValueNanos(System.nanoTime() - start));
-                        }
-                    }
-                } catch (Throwable t) {
-                    indexShard.warmerService().logger().warn("failed to warm-up norms", t);
-                } finally {
-                    latch.countDown();
-                }
-            });
-
-            return () -> latch.await();
-        }
-
-        @Override
-        public TerminationHandle warmTopReader(IndexShard indexShard, final Engine.Searcher searcher) {
-            return TerminationHandle.NO_WAIT;
-        }
+        TerminationHandle warmReader(IndexShard indexShard, Engine.Searcher searcher);
     }
 
     private static class FieldDataWarmer implements IndexWarmer.Listener {
@@ -203,67 +116,17 @@ public final class IndexWarmer extends AbstractComponent {
         }
 
         @Override
-        public TerminationHandle warmNewReaders(final IndexShard indexShard, final Engine.Searcher searcher) {
-            final MapperService mapperService = indexShard.mapperService();
-            final Map<String, MappedFieldType> warmUp = new HashMap<>();
-            for (DocumentMapper docMapper : mapperService.docMappers(false)) {
-                for (FieldMapper fieldMapper : docMapper.mappers()) {
-                    final FieldDataType fieldDataType = fieldMapper.fieldType().fieldDataType();
-                    final String indexName = fieldMapper.fieldType().name();
-                    if (fieldDataType == null) {
-                        continue;
-                    }
-                    if (fieldDataType.getLoading() == MappedFieldType.Loading.LAZY) {
-                        continue;
-                    }
-
-                    if (warmUp.containsKey(indexName)) {
-                        continue;
-                    }
-                    warmUp.put(indexName, fieldMapper.fieldType());
-                }
-            }
-            final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService();
-            final CountDownLatch latch = new CountDownLatch(searcher.reader().leaves().size() * warmUp.size());
-            for (final LeafReaderContext ctx : searcher.reader().leaves()) {
-                for (final MappedFieldType fieldType : warmUp.values()) {
-                    executor.execute(() -> {
-                        try {
-                            final long start = System.nanoTime();
-                            indexFieldDataService.getForField(fieldType).load(ctx);
-                            if (indexShard.warmerService().logger().isTraceEnabled()) {
-                                indexShard.warmerService().logger().trace("warmed fielddata for [{}], took [{}]", fieldType.name(),
-                                    TimeValue.timeValueNanos(System.nanoTime() - start));
-                            }
-                        } catch (Throwable t) {
-                            indexShard.warmerService().logger().warn("failed to warm-up fielddata for [{}]", t, fieldType.name());
-                        } finally {
-                            latch.countDown();
-                        }
-                    });
-                }
-            }
-            return () -> latch.await();
-        }
-
-        @Override
-        public TerminationHandle warmTopReader(final IndexShard indexShard, final Engine.Searcher searcher) {
+        public TerminationHandle warmReader(final IndexShard indexShard, final Engine.Searcher searcher) {
             final MapperService mapperService = indexShard.mapperService();
             final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>();
             for (DocumentMapper docMapper : mapperService.docMappers(false)) {
                 for (FieldMapper fieldMapper : docMapper.mappers()) {
-                    final FieldDataType fieldDataType = fieldMapper.fieldType().fieldDataType();
-                    final String indexName = fieldMapper.fieldType().name();
-                    if (fieldDataType == null) {
+                    final MappedFieldType fieldType = fieldMapper.fieldType();
+                    final String indexName = fieldType.name();
+                    if (fieldType.eagerGlobalOrdinals() == false) {
                         continue;
                     }
-                    if (fieldDataType.getLoading() != MappedFieldType.Loading.EAGER_GLOBAL_ORDINALS) {
-                        continue;
-                    }
-                    if (warmUpGlobalOrdinals.containsKey(indexName)) {
-                        continue;
-                    }
-                    warmUpGlobalOrdinals.put(indexName, fieldMapper.fieldType());
+                    warmUpGlobalOrdinals.put(indexName, fieldType);
                 }
             }
             final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService();
@@ -273,7 +136,12 @@ public final class IndexWarmer extends AbstractComponent {
                     try {
                         final long start = System.nanoTime();
                         IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType);
-                        ifd.loadGlobal(searcher.getDirectoryReader());
+                        DirectoryReader reader = searcher.getDirectoryReader();
+                        IndexFieldData<?> global = ifd.loadGlobal(reader);
+                        if (reader.leaves().isEmpty() == false) {
+                            global.load(reader.leaves().get(0));
+                        }
+
                         if (indexShard.warmerService().logger().isTraceEnabled()) {
                             indexShard.warmerService().logger().trace("warmed global ordinals for [{}], took [{}]", fieldType.name(),
                                 TimeValue.timeValueNanos(System.nanoTime() - start));
diff --git a/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java b/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
index 5452daa7f07..ff10179f026 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.engine.Engine;
@@ -36,6 +37,7 @@ import java.util.concurrent.TimeUnit;
 /**
  */
 public final class IndexingSlowLog implements IndexingOperationListener {
+    private final Index index;
     private boolean reformat;
     private long indexWarnThreshold;
     private long indexInfoThreshold;
@@ -51,15 +53,25 @@ public final class IndexingSlowLog implements IndexingOperationListener {
     private SlowLogLevel level;
 
     private final ESLogger indexLogger;
-    private final ESLogger deleteLogger;
 
     private static final String INDEX_INDEXING_SLOWLOG_PREFIX = "index.indexing.slowlog";
-    public static final Setting<TimeValue> INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN_SETTING = Setting.timeSetting(INDEX_INDEXING_SLOWLOG_PREFIX +".threshold.index.warn", TimeValue.timeValueNanos(-1), TimeValue.timeValueMillis(-1), true, Setting.Scope.INDEX);
-    public static final Setting<TimeValue> INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO_SETTING = Setting.timeSetting(INDEX_INDEXING_SLOWLOG_PREFIX +".threshold.index.info", TimeValue.timeValueNanos(-1), TimeValue.timeValueMillis(-1), true, Setting.Scope.INDEX);
-    public static final Setting<TimeValue> INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG_SETTING = Setting.timeSetting(INDEX_INDEXING_SLOWLOG_PREFIX +".threshold.index.debug", TimeValue.timeValueNanos(-1), TimeValue.timeValueMillis(-1), true, Setting.Scope.INDEX);
-    public static final Setting<TimeValue> INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE_SETTING = Setting.timeSetting(INDEX_INDEXING_SLOWLOG_PREFIX +".threshold.index.trace", TimeValue.timeValueNanos(-1), TimeValue.timeValueMillis(-1), true, Setting.Scope.INDEX);
-    public static final Setting<Boolean> INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING = Setting.boolSetting(INDEX_INDEXING_SLOWLOG_PREFIX +".reformat", true, true, Setting.Scope.INDEX);
-    public static final Setting<SlowLogLevel> INDEX_INDEXING_SLOWLOG_LEVEL_SETTING = new Setting<>(INDEX_INDEXING_SLOWLOG_PREFIX +".level", SlowLogLevel.TRACE.name(), SlowLogLevel::parse, true, Setting.Scope.INDEX);
+    public static final Setting<TimeValue> INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN_SETTING =
+        Setting.timeSetting(INDEX_INDEXING_SLOWLOG_PREFIX +".threshold.index.warn", TimeValue.timeValueNanos(-1),
+            TimeValue.timeValueMillis(-1), Property.Dynamic, Property.IndexScope);
+    public static final Setting<TimeValue> INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO_SETTING =
+        Setting.timeSetting(INDEX_INDEXING_SLOWLOG_PREFIX +".threshold.index.info", TimeValue.timeValueNanos(-1),
+            TimeValue.timeValueMillis(-1), Property.Dynamic, Property.IndexScope);
+    public static final Setting<TimeValue> INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG_SETTING =
+        Setting.timeSetting(INDEX_INDEXING_SLOWLOG_PREFIX +".threshold.index.debug", TimeValue.timeValueNanos(-1),
+            TimeValue.timeValueMillis(-1), Property.Dynamic, Property.IndexScope);
+    public static final Setting<TimeValue> INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE_SETTING =
+        Setting.timeSetting(INDEX_INDEXING_SLOWLOG_PREFIX +".threshold.index.trace", TimeValue.timeValueNanos(-1),
+            TimeValue.timeValueMillis(-1), Property.Dynamic, Property.IndexScope);
+    public static final Setting<Boolean> INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING =
+        Setting.boolSetting(INDEX_INDEXING_SLOWLOG_PREFIX +".reformat", true, Property.Dynamic, Property.IndexScope);
+    public static final Setting<SlowLogLevel> INDEX_INDEXING_SLOWLOG_LEVEL_SETTING =
+        new Setting<>(INDEX_INDEXING_SLOWLOG_PREFIX +".level", SlowLogLevel.TRACE.name(), SlowLogLevel::parse, Property.Dynamic,
+            Property.IndexScope);
     /**
      * Reads how much of the source to log. The user can specify any value they
      * like and numbers are interpreted the maximum number of characters to log
@@ -72,19 +84,11 @@ public final class IndexingSlowLog implements IndexingOperationListener {
         } catch (NumberFormatException e) {
             return Booleans.parseBoolean(value, true) ? Integer.MAX_VALUE : 0;
         }
-    }, true, Setting.Scope.INDEX);
+    }, Property.Dynamic, Property.IndexScope);
 
     IndexingSlowLog(IndexSettings indexSettings) {
-        this(indexSettings, Loggers.getLogger(INDEX_INDEXING_SLOWLOG_PREFIX + ".index"),
-                Loggers.getLogger(INDEX_INDEXING_SLOWLOG_PREFIX + ".delete"));
-    }
-
-    /**
-     * Build with the specified loggers. Only used to testing.
-     */
-    IndexingSlowLog(IndexSettings indexSettings, ESLogger indexLogger, ESLogger deleteLogger) {
-        this.indexLogger = indexLogger;
-        this.deleteLogger = deleteLogger;
+        this.indexLogger = Loggers.getLogger(INDEX_INDEXING_SLOWLOG_PREFIX + ".index", indexSettings.getSettings());
+        this.index = indexSettings.getIndex();
 
         indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING, this::setReformat);
         this.reformat = indexSettings.getValue(INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING);
@@ -109,7 +113,6 @@ public final class IndexingSlowLog implements IndexingOperationListener {
     private void setLevel(SlowLogLevel level) {
         this.level = level;
         this.indexLogger.setLevel(level.name());
-        this.deleteLogger.setLevel(level.name());
     }
 
     private void setWarnThreshold(TimeValue warnThreshold) {
@@ -141,13 +144,13 @@ public final class IndexingSlowLog implements IndexingOperationListener {
 
     private void postIndexing(ParsedDocument doc, long tookInNanos) {
         if (indexWarnThreshold >= 0 && tookInNanos > indexWarnThreshold) {
-            indexLogger.warn("{}", new SlowLogParsedDocumentPrinter(doc, tookInNanos, reformat, maxSourceCharsToLog));
+            indexLogger.warn("{}", new SlowLogParsedDocumentPrinter(index, doc, tookInNanos, reformat, maxSourceCharsToLog));
         } else if (indexInfoThreshold >= 0 && tookInNanos > indexInfoThreshold) {
-            indexLogger.info("{}", new SlowLogParsedDocumentPrinter(doc, tookInNanos, reformat, maxSourceCharsToLog));
+            indexLogger.info("{}", new SlowLogParsedDocumentPrinter(index, doc, tookInNanos, reformat, maxSourceCharsToLog));
         } else if (indexDebugThreshold >= 0 && tookInNanos > indexDebugThreshold) {
-            indexLogger.debug("{}", new SlowLogParsedDocumentPrinter(doc, tookInNanos, reformat, maxSourceCharsToLog));
+            indexLogger.debug("{}", new SlowLogParsedDocumentPrinter(index, doc, tookInNanos, reformat, maxSourceCharsToLog));
         } else if (indexTraceThreshold >= 0 && tookInNanos > indexTraceThreshold) {
-            indexLogger.trace("{}", new SlowLogParsedDocumentPrinter(doc, tookInNanos, reformat, maxSourceCharsToLog));
+            indexLogger.trace("{}", new SlowLogParsedDocumentPrinter(index, doc, tookInNanos, reformat, maxSourceCharsToLog));
         }
     }
 
@@ -156,9 +159,11 @@ public final class IndexingSlowLog implements IndexingOperationListener {
         private final long tookInNanos;
         private final boolean reformat;
         private final int maxSourceCharsToLog;
+        private final Index index;
 
-        SlowLogParsedDocumentPrinter(ParsedDocument doc, long tookInNanos, boolean reformat, int maxSourceCharsToLog) {
+        SlowLogParsedDocumentPrinter(Index index, ParsedDocument doc, long tookInNanos, boolean reformat, int maxSourceCharsToLog) {
             this.doc = doc;
+            this.index = index;
             this.tookInNanos = tookInNanos;
             this.reformat = reformat;
             this.maxSourceCharsToLog = maxSourceCharsToLog;
@@ -167,6 +172,7 @@ public final class IndexingSlowLog implements IndexingOperationListener {
         @Override
         public String toString() {
             StringBuilder sb = new StringBuilder();
+            sb.append(index).append(" ");
             sb.append("took[").append(TimeValue.timeValueNanos(tookInNanos)).append("], took_millis[").append(TimeUnit.NANOSECONDS.toMillis(tookInNanos)).append("], ");
             sb.append("type[").append(doc.type()).append("], ");
             sb.append("id[").append(doc.id()).append("], ");
diff --git a/core/src/main/java/org/elasticsearch/index/MergePolicyConfig.java b/core/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
index fc9f30cf3fd..c8d82eae888 100644
--- a/core/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
+++ b/core/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
@@ -25,6 +25,7 @@ import org.apache.lucene.index.TieredMergePolicy;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 
@@ -126,15 +127,31 @@ public final class MergePolicyConfig {
     public static final ByteSizeValue   DEFAULT_MAX_MERGED_SEGMENT          = new ByteSizeValue(5, ByteSizeUnit.GB);
     public static final double          DEFAULT_SEGMENTS_PER_TIER           = 10.0d;
     public static final double          DEFAULT_RECLAIM_DELETES_WEIGHT      = 2.0d;
-    public static final Setting<Double> INDEX_COMPOUND_FORMAT_SETTING       = new Setting<>("index.compound_format", Double.toString(TieredMergePolicy.DEFAULT_NO_CFS_RATIO), MergePolicyConfig::parseNoCFSRatio, true, Setting.Scope.INDEX);
+    public static final Setting<Double> INDEX_COMPOUND_FORMAT_SETTING       =
+        new Setting<>("index.compound_format", Double.toString(TieredMergePolicy.DEFAULT_NO_CFS_RATIO), MergePolicyConfig::parseNoCFSRatio,
+            Property.Dynamic, Property.IndexScope);
 
-    public static final Setting<Double> INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING = Setting.doubleSetting("index.merge.policy.expunge_deletes_allowed", DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d, true, Setting.Scope.INDEX);
-    public static final Setting<ByteSizeValue> INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING = Setting.byteSizeSetting("index.merge.policy.floor_segment", DEFAULT_FLOOR_SEGMENT, true, Setting.Scope.INDEX);
-    public static final Setting<Integer> INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING = Setting.intSetting("index.merge.policy.max_merge_at_once", DEFAULT_MAX_MERGE_AT_ONCE, 2, true, Setting.Scope.INDEX);
-    public static final Setting<Integer> INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING = Setting.intSetting("index.merge.policy.max_merge_at_once_explicit", DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT, 2, true, Setting.Scope.INDEX);
-    public static final Setting<ByteSizeValue> INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING = Setting.byteSizeSetting("index.merge.policy.max_merged_segment", DEFAULT_MAX_MERGED_SEGMENT, true, Setting.Scope.INDEX);
-    public static final Setting<Double> INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING = Setting.doubleSetting("index.merge.policy.segments_per_tier", DEFAULT_SEGMENTS_PER_TIER, 2.0d, true, Setting.Scope.INDEX);
-    public static final Setting<Double> INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING = Setting.doubleSetting("index.merge.policy.reclaim_deletes_weight", DEFAULT_RECLAIM_DELETES_WEIGHT, 0.0d, true, Setting.Scope.INDEX);
+    public static final Setting<Double> INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING =
+        Setting.doubleSetting("index.merge.policy.expunge_deletes_allowed", DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d,
+            Property.Dynamic, Property.IndexScope);
+    public static final Setting<ByteSizeValue> INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING =
+        Setting.byteSizeSetting("index.merge.policy.floor_segment", DEFAULT_FLOOR_SEGMENT,
+            Property.Dynamic, Property.IndexScope);
+    public static final Setting<Integer> INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING =
+        Setting.intSetting("index.merge.policy.max_merge_at_once", DEFAULT_MAX_MERGE_AT_ONCE, 2,
+            Property.Dynamic, Property.IndexScope);
+    public static final Setting<Integer> INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING =
+        Setting.intSetting("index.merge.policy.max_merge_at_once_explicit", DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT, 2,
+            Property.Dynamic, Property.IndexScope);
+    public static final Setting<ByteSizeValue> INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING =
+        Setting.byteSizeSetting("index.merge.policy.max_merged_segment", DEFAULT_MAX_MERGED_SEGMENT,
+            Property.Dynamic, Property.IndexScope);
+    public static final Setting<Double> INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING =
+        Setting.doubleSetting("index.merge.policy.segments_per_tier", DEFAULT_SEGMENTS_PER_TIER, 2.0d,
+            Property.Dynamic, Property.IndexScope);
+    public static final Setting<Double> INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING =
+        Setting.doubleSetting("index.merge.policy.reclaim_deletes_weight", DEFAULT_RECLAIM_DELETES_WEIGHT, 0.0d,
+            Property.Dynamic, Property.IndexScope);
     public static final String INDEX_MERGE_ENABLED = "index.merge.enabled"; // don't convert to Setting<> and register... we only set this in tests and register via a plugin
 
 
diff --git a/core/src/main/java/org/elasticsearch/index/MergeSchedulerConfig.java b/core/src/main/java/org/elasticsearch/index/MergeSchedulerConfig.java
index 0d212a4eb30..2eb43a50ee4 100644
--- a/core/src/main/java/org/elasticsearch/index/MergeSchedulerConfig.java
+++ b/core/src/main/java/org/elasticsearch/index/MergeSchedulerConfig.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index;
 
 import org.apache.lucene.index.ConcurrentMergeScheduler;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 
 /**
@@ -51,9 +52,17 @@ import org.elasticsearch.common.util.concurrent.EsExecutors;
  */
 public final class MergeSchedulerConfig {
 
-    public static final Setting<Integer> MAX_THREAD_COUNT_SETTING = new Setting<>("index.merge.scheduler.max_thread_count", (s) -> Integer.toString(Math.max(1, Math.min(4, EsExecutors.boundedNumberOfProcessors(s) / 2))), (s) -> Setting.parseInt(s, 1, "index.merge.scheduler.max_thread_count"), true, Setting.Scope.INDEX);
-    public static final Setting<Integer> MAX_MERGE_COUNT_SETTING = new Setting<>("index.merge.scheduler.max_merge_count", (s) -> Integer.toString(MAX_THREAD_COUNT_SETTING.get(s) + 5), (s) -> Setting.parseInt(s, 1, "index.merge.scheduler.max_merge_count"), true, Setting.Scope.INDEX);
-    public static final Setting<Boolean> AUTO_THROTTLE_SETTING = Setting.boolSetting("index.merge.scheduler.auto_throttle", true, true, Setting.Scope.INDEX);
+    public static final Setting<Integer> MAX_THREAD_COUNT_SETTING =
+        new Setting<>("index.merge.scheduler.max_thread_count",
+            (s) -> Integer.toString(Math.max(1, Math.min(4, EsExecutors.boundedNumberOfProcessors(s) / 2))),
+            (s) -> Setting.parseInt(s, 1, "index.merge.scheduler.max_thread_count"), Property.Dynamic,
+            Property.IndexScope);
+    public static final Setting<Integer> MAX_MERGE_COUNT_SETTING =
+        new Setting<>("index.merge.scheduler.max_merge_count",
+            (s) -> Integer.toString(MAX_THREAD_COUNT_SETTING.get(s) + 5),
+            (s) -> Setting.parseInt(s, 1, "index.merge.scheduler.max_merge_count"), Property.Dynamic, Property.IndexScope);
+    public static final Setting<Boolean> AUTO_THROTTLE_SETTING =
+        Setting.boolSetting("index.merge.scheduler.auto_throttle", true, Property.Dynamic, Property.IndexScope);
 
     private volatile boolean autoThrottle;
     private volatile int maxThreadCount;
diff --git a/core/src/main/java/org/elasticsearch/index/SearchSlowLog.java b/core/src/main/java/org/elasticsearch/index/SearchSlowLog.java
index df3139fe57c..cfa779d64aa 100644
--- a/core/src/main/java/org/elasticsearch/index/SearchSlowLog.java
+++ b/core/src/main/java/org/elasticsearch/index/SearchSlowLog.java
@@ -23,6 +23,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -50,16 +51,35 @@ public final class SearchSlowLog {
     private final ESLogger fetchLogger;
 
     private static final String INDEX_SEARCH_SLOWLOG_PREFIX = "index.search.slowlog";
-    public static final Setting<TimeValue> INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING = Setting.timeSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.query.warn", TimeValue.timeValueNanos(-1), TimeValue.timeValueMillis(-1), true, Setting.Scope.INDEX);
-    public static final Setting<TimeValue> INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO_SETTING = Setting.timeSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.query.info", TimeValue.timeValueNanos(-1), TimeValue.timeValueMillis(-1), true, Setting.Scope.INDEX);
-    public static final Setting<TimeValue> INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG_SETTING = Setting.timeSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.query.debug", TimeValue.timeValueNanos(-1), TimeValue.timeValueMillis(-1), true, Setting.Scope.INDEX);
-    public static final Setting<TimeValue> INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE_SETTING = Setting.timeSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.query.trace", TimeValue.timeValueNanos(-1), TimeValue.timeValueMillis(-1), true, Setting.Scope.INDEX);
-    public static final Setting<TimeValue> INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING = Setting.timeSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.fetch.warn", TimeValue.timeValueNanos(-1), TimeValue.timeValueMillis(-1), true, Setting.Scope.INDEX);
-    public static final Setting<TimeValue> INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING = Setting.timeSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.fetch.info", TimeValue.timeValueNanos(-1), TimeValue.timeValueMillis(-1), true, Setting.Scope.INDEX);
-    public static final Setting<TimeValue> INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING = Setting.timeSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.fetch.debug", TimeValue.timeValueNanos(-1), TimeValue.timeValueMillis(-1), true, Setting.Scope.INDEX);
-    public static final Setting<TimeValue> INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE_SETTING = Setting.timeSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.fetch.trace", TimeValue.timeValueNanos(-1), TimeValue.timeValueMillis(-1), true, Setting.Scope.INDEX);
-    public static final Setting<Boolean> INDEX_SEARCH_SLOWLOG_REFORMAT = Setting.boolSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".reformat", true, true, Setting.Scope.INDEX);
-    public static final Setting<SlowLogLevel> INDEX_SEARCH_SLOWLOG_LEVEL = new Setting<>(INDEX_SEARCH_SLOWLOG_PREFIX + ".level", SlowLogLevel.TRACE.name(), SlowLogLevel::parse, true, Setting.Scope.INDEX);
+    public static final Setting<TimeValue> INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING =
+        Setting.timeSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.query.warn", TimeValue.timeValueNanos(-1),
+            TimeValue.timeValueMillis(-1), Property.Dynamic, Property.IndexScope);
+    public static final Setting<TimeValue> INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO_SETTING =
+        Setting.timeSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.query.info", TimeValue.timeValueNanos(-1),
+            TimeValue.timeValueMillis(-1), Property.Dynamic, Property.IndexScope);
+    public static final Setting<TimeValue> INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG_SETTING =
+        Setting.timeSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.query.debug", TimeValue.timeValueNanos(-1),
+            TimeValue.timeValueMillis(-1), Property.Dynamic, Property.IndexScope);
+    public static final Setting<TimeValue> INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE_SETTING =
+        Setting.timeSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.query.trace", TimeValue.timeValueNanos(-1),
+            TimeValue.timeValueMillis(-1), Property.Dynamic, Property.IndexScope);
+    public static final Setting<TimeValue> INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING =
+        Setting.timeSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.fetch.warn", TimeValue.timeValueNanos(-1),
+            TimeValue.timeValueMillis(-1), Property.Dynamic, Property.IndexScope);
+    public static final Setting<TimeValue> INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING =
+        Setting.timeSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.fetch.info", TimeValue.timeValueNanos(-1),
+            TimeValue.timeValueMillis(-1), Property.Dynamic, Property.IndexScope);
+    public static final Setting<TimeValue> INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING =
+        Setting.timeSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.fetch.debug", TimeValue.timeValueNanos(-1),
+            TimeValue.timeValueMillis(-1), Property.Dynamic, Property.IndexScope);
+    public static final Setting<TimeValue> INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE_SETTING =
+        Setting.timeSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".threshold.fetch.trace", TimeValue.timeValueNanos(-1),
+            TimeValue.timeValueMillis(-1), Property.Dynamic, Property.IndexScope);
+    public static final Setting<Boolean> INDEX_SEARCH_SLOWLOG_REFORMAT =
+        Setting.boolSetting(INDEX_SEARCH_SLOWLOG_PREFIX + ".reformat", true, Property.Dynamic, Property.IndexScope);
+    public static final Setting<SlowLogLevel> INDEX_SEARCH_SLOWLOG_LEVEL =
+        new Setting<>(INDEX_SEARCH_SLOWLOG_PREFIX + ".level", SlowLogLevel.TRACE.name(), SlowLogLevel::parse, Property.Dynamic,
+            Property.IndexScope);
 
     public SearchSlowLog(IndexSettings indexSettings) {
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/Analysis.java b/core/src/main/java/org/elasticsearch/index/analysis/Analysis.java
index b7481e78496..1054721535e 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/Analysis.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/Analysis.java
@@ -67,8 +67,10 @@ import org.elasticsearch.env.Environment;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.Reader;
+import java.nio.charset.CharacterCodingException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -163,7 +165,8 @@ public class Analysis {
         NAMED_STOP_WORDS = unmodifiableMap(namedStopWords);
     }
 
-    public static CharArraySet parseWords(Environment env, Settings settings, String name, CharArraySet defaultWords, Map<String, Set<?>> namedWords, boolean ignoreCase) {
+    public static CharArraySet parseWords(Environment env, Settings settings, String name, CharArraySet defaultWords,
+                                          Map<String, Set<?>> namedWords, boolean ignoreCase) {
         String value = settings.get(name);
         if (value != null) {
             if ("_none_".equals(value)) {
@@ -237,12 +240,17 @@ public class Analysis {
             }
         }
 
-        final Path wordListFile = env.configFile().resolve(wordListPath);
+        final Path path = env.configFile().resolve(wordListPath);
 
-        try (BufferedReader reader = FileSystemUtils.newBufferedReader(wordListFile.toUri().toURL(), StandardCharsets.UTF_8)) {
+        try (BufferedReader reader = FileSystemUtils.newBufferedReader(path.toUri().toURL(), StandardCharsets.UTF_8)) {
             return loadWordList(reader, "#");
+        } catch (CharacterCodingException ex) {
+            String message = String.format(Locale.ROOT,
+                "Unsupported character encoding detected while reading %s_path: %s - files must be UTF-8 encoded",
+                settingPrefix, path.toString());
+            throw new IllegalArgumentException(message, ex);
         } catch (IOException ioe) {
-            String message = String.format(Locale.ROOT, "IOException while reading %s_path: %s", settingPrefix);
+            String message = String.format(Locale.ROOT, "IOException while reading %s_path: %s", settingPrefix, path.toString());
             throw new IllegalArgumentException(message, ioe);
         }
     }
@@ -256,7 +264,7 @@ public class Analysis {
             } else {
                 br = new BufferedReader(reader);
             }
-            String word = null;
+            String word;
             while ((word = br.readLine()) != null) {
                 if (!Strings.hasText(word)) {
                     continue;
@@ -283,13 +291,16 @@ public class Analysis {
         if (filePath == null) {
             return null;
         }
-
         final Path path = env.configFile().resolve(filePath);
-
         try {
             return FileSystemUtils.newBufferedReader(path.toUri().toURL(), StandardCharsets.UTF_8);
+        } catch (CharacterCodingException ex) {
+            String message = String.format(Locale.ROOT,
+                "Unsupported character encoding detected while reading %s_path: %s files must be UTF-8 encoded",
+                settingPrefix, path.toString());
+            throw new IllegalArgumentException(message, ex);
         } catch (IOException ioe) {
-            String message = String.format(Locale.ROOT, "IOException while reading %s_path: %s", settingPrefix);
+            String message = String.format(Locale.ROOT, "IOException while reading %s_path: %s", settingPrefix, path.toString());
             throw new IllegalArgumentException(message, ioe);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
index a8a7b4fe004..3c2d6bfb260 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
@@ -85,6 +85,10 @@ public final class AnalysisRegistry implements Closeable {
         this.analyzers = Collections.unmodifiableMap(analyzerBuilder);
     }
 
+    public HunspellService getHunspellService() {
+        return hunspellService;
+    }
+
     /**
      * Returns a registered {@link TokenizerFactory} provider by name or <code>null</code> if the tokenizer was not registered
      */
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java
index 09e96f3743b..b9146df8c96 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java
@@ -127,7 +127,7 @@ public class AnalysisService extends AbstractIndexComponent implements Closeable
         }
         if (analyzers.containsKey("default_index")) {
             final Version createdVersion = indexSettings.getIndexVersionCreated();
-            if (createdVersion.onOrAfter(Version.V_5_0_0)) {
+            if (createdVersion.onOrAfter(Version.V_5_0_0_alpha1)) {
                 throw new IllegalArgumentException("setting [index.analysis.analyzer.default_index] is not supported anymore, use [index.analysis.analyzer.default] instead for index [" + index().getName() + "]");
             } else {
                 deprecationLogger.deprecated("setting [index.analysis.analyzer.default_index] is deprecated, use [index.analysis.analyzer.default] instead for index [{}]", index().getName());
@@ -155,7 +155,7 @@ public class AnalysisService extends AbstractIndexComponent implements Closeable
                     // because analyzers are aliased, they might be closed several times
                     // an NPE is thrown in this case, so ignore....
                 } catch (Exception e) {
-                    logger.debug("failed to close analyzer " + analyzer);
+                    logger.debug("failed to close analyzer {}", analyzer);
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java b/core/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java
index 25ff8f96834..1dd562c4bb1 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java
@@ -93,7 +93,7 @@ public class NamedAnalyzer extends DelegatingAnalyzerWrapper {
     public String toString() {
         return "analyzer name[" + name + "], analyzer [" + analyzer + "]";
     }
-    
+
     /** It is an error if this is ever used, it means we screwed up! */
     static final ReuseStrategy ERROR_STRATEGY = new Analyzer.ReuseStrategy() {
         @Override
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/NumericDoubleAnalyzer.java b/core/src/main/java/org/elasticsearch/index/analysis/NumericDoubleAnalyzer.java
index e90409421d2..77716e7a43d 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/NumericDoubleAnalyzer.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/NumericDoubleAnalyzer.java
@@ -56,4 +56,4 @@ public class NumericDoubleAnalyzer extends NumericAnalyzer<NumericDoubleTokenize
     protected NumericDoubleTokenizer createNumericTokenizer(char[] buffer) throws IOException {
         return new NumericDoubleTokenizer(precisionStep, buffer);
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/NumericFloatAnalyzer.java b/core/src/main/java/org/elasticsearch/index/analysis/NumericFloatAnalyzer.java
index 85344830fbf..e8ee0295cff 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/NumericFloatAnalyzer.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/NumericFloatAnalyzer.java
@@ -56,4 +56,4 @@ public class NumericFloatAnalyzer extends NumericAnalyzer<NumericFloatTokenizer>
     protected NumericFloatTokenizer createNumericTokenizer(char[] buffer) throws IOException {
         return new NumericFloatTokenizer(precisionStep, buffer);
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/NumericLongAnalyzer.java b/core/src/main/java/org/elasticsearch/index/analysis/NumericLongAnalyzer.java
index ab112396392..9b865920341 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/NumericLongAnalyzer.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/NumericLongAnalyzer.java
@@ -56,4 +56,4 @@ public class NumericLongAnalyzer extends NumericAnalyzer<NumericLongTokenizer> {
     protected NumericLongTokenizer createNumericTokenizer(char[] buffer) throws IOException {
         return new NumericLongTokenizer(precisionStep, buffer);
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java b/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java
index 61733f24695..b41f5bc0125 100644
--- a/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java
+++ b/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java
@@ -24,6 +24,7 @@ import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.cache.query.QueryCache;
+import org.elasticsearch.index.percolator.PercolatorQueryCache;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -35,11 +36,14 @@ public class IndexCache extends AbstractIndexComponent implements Closeable {
 
     private final QueryCache queryCache;
     private final BitsetFilterCache bitsetFilterCache;
+    private final PercolatorQueryCache percolatorQueryCache;
 
-    public IndexCache(IndexSettings indexSettings, QueryCache queryCache, BitsetFilterCache bitsetFilterCache) {
+    public IndexCache(IndexSettings indexSettings, QueryCache queryCache, BitsetFilterCache bitsetFilterCache,
+                      PercolatorQueryCache percolatorQueryCache) {
         super(indexSettings);
         this.queryCache = queryCache;
         this.bitsetFilterCache = bitsetFilterCache;
+        this.percolatorQueryCache = percolatorQueryCache;
     }
 
     public QueryCache query() {
@@ -53,9 +57,13 @@ public class IndexCache extends AbstractIndexComponent implements Closeable {
         return bitsetFilterCache;
     }
 
+    public PercolatorQueryCache getPercolatorQueryCache() {
+        return percolatorQueryCache;
+    }
+
     @Override
     public void close() throws IOException {
-        IOUtils.close(queryCache, bitsetFilterCache);
+        IOUtils.close(queryCache, bitsetFilterCache, percolatorQueryCache);
     }
 
     public void clear(String reason) {
diff --git a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
index f7802330ab7..2452e8147c2 100644
--- a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
+++ b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
@@ -38,6 +38,7 @@ import org.elasticsearch.common.cache.RemovalListener;
 import org.elasticsearch.common.cache.RemovalNotification;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
@@ -70,7 +71,8 @@ import java.util.concurrent.Executor;
  */
 public final class BitsetFilterCache extends AbstractIndexComponent implements LeafReader.CoreClosedListener, RemovalListener<Object, Cache<Query, BitsetFilterCache.Value>>, Closeable {
 
-    public static final Setting<Boolean> INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING = Setting.boolSetting("index.load_fixed_bitset_filters_eagerly", true, false, Setting.Scope.INDEX);
+    public static final Setting<Boolean> INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING =
+        Setting.boolSetting("index.load_fixed_bitset_filters_eagerly", true, Property.IndexScope);
 
     private final boolean loadRandomAccessFiltersEagerly;
     private final Cache<Object, Cache<Query, Value>> loadedFilters;
@@ -214,7 +216,7 @@ public final class BitsetFilterCache extends AbstractIndexComponent implements L
         }
 
         @Override
-        public IndexWarmer.TerminationHandle warmNewReaders(final IndexShard indexShard, final Engine.Searcher searcher) {
+        public IndexWarmer.TerminationHandle warmReader(final IndexShard indexShard, final Engine.Searcher searcher) {
             if (indexSettings.getIndex().equals(indexShard.indexSettings().getIndex()) == false) {
                 // this is from a different index
                 return TerminationHandle.NO_WAIT;
@@ -266,11 +268,6 @@ public final class BitsetFilterCache extends AbstractIndexComponent implements L
             return () -> latch.await();
         }
 
-        @Override
-        public TerminationHandle warmTopReader(IndexShard indexShard, Engine.Searcher searcher) {
-            return TerminationHandle.NO_WAIT;
-        }
-
     }
 
     Cache<Object, Cache<Query, Value>> getLoadedFilters() {
diff --git a/core/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java b/core/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java
index 6dd710e4e89..965a2e58f9c 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java
@@ -129,9 +129,9 @@ class ElasticsearchConcurrentMergeScheduler extends ConcurrentMergeScheduler {
                                            merge.rateLimiter.getMBPerSec());
 
             if (tookMS > 20000) { // if more than 20 seconds, DEBUG log it
-                logger.debug(message);
+                logger.debug("{}", message);
             } else if (logger.isTraceEnabled()) {
-                logger.trace(message);
+                logger.trace("{}", message);
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
index 3c5583440e0..c66073bd91f 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -671,7 +671,7 @@ public abstract class Engine implements Closeable {
                     closeNoLock("engine failed on: [" + reason + "]");
                 } finally {
                     if (failedEngine != null) {
-                        logger.debug("tried to fail engine but engine is already failed. ignoring. [{}]", reason, failure);
+                        logger.debug("tried to fail engine but engine is already failed. ignoring. [{}]", failure, reason);
                         return;
                     }
                     logger.warn("failed engine [{}]", failure, reason);
@@ -697,7 +697,7 @@ public abstract class Engine implements Closeable {
                 store.decRef();
             }
         } else {
-            logger.debug("tried to fail engine but could not acquire lock - engine should be failed by now [{}]", reason, failure);
+            logger.debug("tried to fail engine but could not acquire lock - engine should be failed by now [{}]", failure, reason);
         }
     }
 
@@ -1219,12 +1219,9 @@ public abstract class Engine implements Closeable {
      */
     public interface Warmer {
         /**
-         * Called once a new Searcher is opened.
-         * @param searcher the searcer to warm
-         * @param isTopLevelReader <code>true</code> iff the searcher is build from a top-level reader.
-         *                         Otherwise the searcher might be build from a leaf reader to warm in isolation
+         * Called once a new Searcher is opened on the top-level searcher.
          */
-        void warm(Engine.Searcher searcher, boolean isTopLevelReader);
+        void warm(Engine.Searcher searcher);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
index 9740ccd0358..47001f40309 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
@@ -26,6 +26,7 @@ import org.apache.lucene.search.QueryCache;
 import org.apache.lucene.search.QueryCachingPolicy;
 import org.apache.lucene.search.similarities.Similarity;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -39,7 +40,7 @@ import org.elasticsearch.index.translog.TranslogConfig;
 import org.elasticsearch.indices.IndexingMemoryController;
 import org.elasticsearch.threadpool.ThreadPool;
 
-import java.util.Set;
+import java.util.function.Function;
 
 /*
  * Holds all the configuration that is used to create an {@link Engine}.
@@ -70,20 +71,23 @@ public final class EngineConfig {
     /**
      * Index setting to change the low level lucene codec used for writing new segments.
      * This setting is <b>not</b> realtime updateable.
+     * This setting is also settable on the node and the index level, it's commonly used in hot/cold node archs where index is likely
+     * allocated on both `kind` of nodes.
      */
-    public static final Setting<String> INDEX_CODEC_SETTING = new Setting<>("index.codec", "default", (s) -> {
-        switch(s) {
+    public static final Setting<String> INDEX_CODEC_SETTING = new Setting<>("index.codec", "default", s -> {
+        switch (s) {
             case "default":
             case "best_compression":
             case "lucene_default":
                 return s;
             default:
                 if (Codec.availableCodecs().contains(s) == false) { // we don't error message the not officially supported ones
-                    throw new IllegalArgumentException("unknown value for [index.codec] must be one of [default, best_compression] but was: " + s);
+                    throw new IllegalArgumentException(
+                        "unknown value for [index.codec] must be one of [default, best_compression] but was: " + s);
                 }
                 return s;
         }
-    }, false, Setting.Scope.INDEX);
+    }, Property.IndexScope, Property.NodeScope);
 
     /** if set to true the engine will start even if the translog id in the commit point can not be found */
     public static final String INDEX_FORCE_NEW_TRANSLOG = "index.engine.force_new_translog";
@@ -98,12 +102,13 @@ public final class EngineConfig {
                         IndexSettings indexSettings, Engine.Warmer warmer, Store store, SnapshotDeletionPolicy deletionPolicy,
                         MergePolicy mergePolicy,Analyzer analyzer,
                         Similarity similarity, CodecService codecService, Engine.EventListener eventListener,
-                        TranslogRecoveryPerformer translogRecoveryPerformer, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, TranslogConfig translogConfig, TimeValue flushMergesAfter) {
+                        TranslogRecoveryPerformer translogRecoveryPerformer, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy,
+                        TranslogConfig translogConfig, TimeValue flushMergesAfter) {
         this.shardId = shardId;
         final Settings settings = indexSettings.getSettings();
         this.indexSettings = indexSettings;
         this.threadPool = threadPool;
-        this.warmer = warmer == null ? (a,b) -> {} : warmer;
+        this.warmer = warmer == null ? (a) -> {} : warmer;
         this.store = store;
         this.deletionPolicy = deletionPolicy;
         this.mergePolicy = mergePolicy;
@@ -139,7 +144,8 @@ public final class EngineConfig {
     }
 
     /**
-     * Returns the initial index buffer size. This setting is only read on startup and otherwise controlled by {@link IndexingMemoryController}
+     * Returns the initial index buffer size. This setting is only read on startup and otherwise controlled
+     * by {@link IndexingMemoryController}
      */
     public ByteSizeValue getIndexingBufferSize() {
         return indexingBufferSize;
@@ -147,11 +153,12 @@ public final class EngineConfig {
 
     /**
      * Returns <code>true</code> iff delete garbage collection in the engine should be enabled. This setting is updateable
-     * in realtime and forces a volatile read. Consumers can safely read this value directly go fetch it's latest value. The default is <code>true</code>
+     * in realtime and forces a volatile read. Consumers can safely read this value directly go fetch it's latest value.
+     * The default is <code>true</code>
      * <p>
      *     Engine GC deletion if enabled collects deleted documents from in-memory realtime data structures after a certain amount of
-     *     time ({@link IndexSettings#getGcDeletesInMillis()} if enabled. Before deletes are GCed they will cause re-adding the document that was deleted
-     *     to fail.
+     *     time ({@link IndexSettings#getGcDeletesInMillis()} if enabled. Before deletes are GCed they will cause re-adding the document
+     *     that was deleted to fail.
      * </p>
      */
     public boolean isEnableGcDeletes() {
@@ -169,7 +176,8 @@ public final class EngineConfig {
     }
 
     /**
-     * Returns a thread-pool mainly used to get estimated time stamps from {@link org.elasticsearch.threadpool.ThreadPool#estimatedTimeInMillis()} and to schedule
+     * Returns a thread-pool mainly used to get estimated time stamps from
+     * {@link org.elasticsearch.threadpool.ThreadPool#estimatedTimeInMillis()} and to schedule
      * async force merge calls on the {@link org.elasticsearch.threadpool.ThreadPool.Names#FORCE_MERGE} thread-pool
      */
     public ThreadPool getThreadPool() {
@@ -184,8 +192,9 @@ public final class EngineConfig {
     }
 
     /**
-     * Returns the {@link org.elasticsearch.index.store.Store} instance that provides access to the {@link org.apache.lucene.store.Directory}
-     * used for the engines {@link org.apache.lucene.index.IndexWriter} to write it's index files to.
+     * Returns the {@link org.elasticsearch.index.store.Store} instance that provides access to the
+     * {@link org.apache.lucene.store.Directory} used for the engines {@link org.apache.lucene.index.IndexWriter} to write it's index files
+     * to.
      * <p>
      * Note: In order to use this instance the consumer needs to increment the stores reference before it's used the first time and hold
      * it's reference until it's not needed anymore.
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 01f02025aeb..aa62f255bb4 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -24,13 +24,10 @@ import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexFormatTooOldException;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.LiveIndexWriterConfig;
 import org.apache.lucene.index.MergePolicy;
-import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.index.SegmentCommitInfo;
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.index.Term;
@@ -51,7 +48,6 @@ import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.lucene.LoggerInfoStream;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
-import org.elasticsearch.common.lucene.index.ElasticsearchLeafReader;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.math.MathUtils;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -70,7 +66,6 @@ import org.elasticsearch.index.translog.TranslogCorruptedException;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -91,7 +86,6 @@ public class InternalEngine extends Engine {
      */
     private volatile long lastDeleteVersionPruneTimeMSec;
 
-    private final Engine.Warmer warmer;
     private final Translog translog;
     private final ElasticsearchConcurrentMergeScheduler mergeScheduler;
 
@@ -131,7 +125,6 @@ public class InternalEngine extends Engine {
         boolean success = false;
         try {
             this.lastDeleteVersionPruneTimeMSec = engineConfig.getThreadPool().estimatedTimeInMillis();
-            this.warmer = engineConfig.getWarmer();
             mergeScheduler = scheduler = new EngineMergeScheduler(engineConfig.getShardId(), engineConfig.getIndexSettings());
             this.dirtyLocks = new Object[Runtime.getRuntime().availableProcessors() * 10]; // we multiply it to have enough...
             for (int i = 0; i < dirtyLocks.length; i++) {
@@ -743,7 +736,9 @@ public class InternalEngine extends Engine {
                     indexWriter.forceMerge(maxNumSegments, true /* blocks and waits for merges*/);
                 }
                 if (flush) {
-                    flush(true, true);
+                    if (tryRenewSyncCommit() == false) {
+                        flush(false, true);
+                    }
                 }
                 if (upgrade) {
                     logger.info("finished segment upgrade");
@@ -929,30 +924,6 @@ public class InternalEngine extends Engine {
             iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().mbFrac());
             iwc.setCodec(engineConfig.getCodec());
             iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh
-            // Warm-up hook for newly-merged segments. Warming up segments here is better since it will be performed at the end
-            // of the merge operation and won't slow down _refresh
-            iwc.setMergedSegmentWarmer(new IndexReaderWarmer() {
-                @Override
-                public void warm(LeafReader reader) throws IOException {
-                    try {
-                        LeafReader esLeafReader = new ElasticsearchLeafReader(reader, shardId);
-                        assert isMergedSegment(esLeafReader);
-                        if (warmer != null) {
-                            final Engine.Searcher searcher = new Searcher("warmer", searcherFactory.newSearcher(esLeafReader, null));
-                            warmer.warm(searcher, false);
-                        }
-                    } catch (Throwable t) {
-                        // Don't fail a merge if the warm-up failed
-                        if (isClosed.get() == false) {
-                            logger.warn("Warm-up failed", t);
-                        }
-                        if (t instanceof Error) {
-                            // assertion/out-of-memory error, don't ignore those
-                            throw (Error) t;
-                        }
-                    }
-                }
-            });
             return new IndexWriter(store.directory(), iwc);
         } catch (LockObtainFailedException ex) {
             logger.warn("could not lock IndexWriter", ex);
@@ -963,14 +934,12 @@ public class InternalEngine extends Engine {
     /** Extended SearcherFactory that warms the segments if needed when acquiring a new searcher */
     final static class SearchFactory extends EngineSearcherFactory {
         private final Engine.Warmer warmer;
-        private final ShardId shardId;
         private final ESLogger logger;
         private final AtomicBoolean isEngineClosed;
 
         SearchFactory(ESLogger logger, AtomicBoolean isEngineClosed, EngineConfig engineConfig) {
             super(engineConfig);
             warmer = engineConfig.getWarmer();
-            shardId = engineConfig.getShardId();
             this.logger = logger;
             this.isEngineClosed = isEngineClosed;
         }
@@ -985,55 +954,13 @@ public class InternalEngine extends Engine {
                 return searcher;
             }
             if (warmer != null) {
-                // we need to pass a custom searcher that does not release anything on Engine.Search Release,
-                // we will release explicitly
-                IndexSearcher newSearcher = null;
-                boolean closeNewSearcher = false;
                 try {
-                    if (previousReader == null) {
-                        // we are starting up - no writer active so we can't acquire a searcher.
-                        newSearcher = searcher;
-                    } else {
-                        // figure out the newSearcher, with only the new readers that are relevant for us
-                        List<IndexReader> readers = new ArrayList<>();
-                        for (LeafReaderContext newReaderContext : reader.leaves()) {
-                            if (isMergedSegment(newReaderContext.reader())) {
-                                // merged segments are already handled by IndexWriterConfig.setMergedSegmentWarmer
-                                continue;
-                            }
-                            boolean found = false;
-                            for (LeafReaderContext currentReaderContext : previousReader.leaves()) {
-                                if (currentReaderContext.reader().getCoreCacheKey().equals(newReaderContext.reader().getCoreCacheKey())) {
-                                    found = true;
-                                    break;
-                                }
-                            }
-                            if (!found) {
-                                readers.add(newReaderContext.reader());
-                            }
-                        }
-                        if (!readers.isEmpty()) {
-                            // we don't want to close the inner readers, just increase ref on them
-                            IndexReader newReader = new MultiReader(readers.toArray(new IndexReader[readers.size()]), false);
-                            newSearcher = super.newSearcher(newReader, null);
-                            closeNewSearcher = true;
-                        }
-                    }
-
-                    if (newSearcher != null) {
-                        warmer.warm(new Searcher("new_reader_warming", newSearcher), false);
-                    }
                     assert searcher.getIndexReader() instanceof ElasticsearchDirectoryReader : "this class needs an ElasticsearchDirectoryReader but got: " + searcher.getIndexReader().getClass();
-                    warmer.warm(new Searcher("top_reader_warming", searcher), true);
+                    warmer.warm(new Searcher("top_reader_warming", searcher));
                 } catch (Throwable e) {
                     if (isEngineClosed.get() == false) {
                         logger.warn("failed to prepare/warm", e);
                     }
-                } finally {
-                    // no need to release the fullSearcher, nothing really is done...
-                    if (newSearcher != null && closeNewSearcher) {
-                        IOUtils.closeWhileHandlingException(newSearcher.getIndexReader()); // ignore
-                    }
                 }
             }
             return searcher;
diff --git a/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java b/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java
index d2c4a3c14c0..10948f35d54 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java
@@ -348,7 +348,7 @@ public class SegmentsStats implements Streamable, ToXContent {
         indexWriterMaxMemoryInBytes = in.readLong();
         bitsetMemoryInBytes = in.readLong();
 
-        if (in.getVersion().onOrAfter(Version.V_5_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) {
             int size = in.readVInt();
             ImmutableOpenMap.Builder<String, Long> map = ImmutableOpenMap.builder(size);
             for (int i = 0; i < size; i++) {
@@ -376,7 +376,7 @@ public class SegmentsStats implements Streamable, ToXContent {
         out.writeLong(indexWriterMaxMemoryInBytes);
         out.writeLong(bitsetMemoryInBytes);
 
-        if (out.getVersion().onOrAfter(Version.V_5_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) {
             out.writeVInt(fileSizes.size());
             for (Iterator<ObjectObjectCursor<String, Long>> it = fileSizes.iterator(); it.hasNext();) {
                 ObjectObjectCursor<String, Long> entry = it.next();
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/FieldDataType.java b/core/src/main/java/org/elasticsearch/index/fielddata/FieldDataType.java
deleted file mode 100644
index 371b802dc0c..00000000000
--- a/core/src/main/java/org/elasticsearch/index/fielddata/FieldDataType.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.fielddata;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.mapper.MappedFieldType.Loading;
-
-/**
- */
-public class FieldDataType {
-
-    public static final String FORMAT_KEY = "format";
-    public static final String DOC_VALUES_FORMAT_VALUE = "doc_values";
-
-    private final String type;
-    private final String typeFormat;
-    private final Loading loading;
-    private final Settings settings;
-
-    public FieldDataType(String type) {
-        this(type, Settings.Builder.EMPTY_SETTINGS);
-    }
-
-    public FieldDataType(String type, Settings.Builder builder) {
-        this(type, builder.build());
-    }
-
-    public FieldDataType(String type, Settings settings) {
-        this.type = type;
-        this.typeFormat = "index.fielddata.type." + type + "." + FORMAT_KEY;
-        this.settings = settings;
-        final String loading = settings.get(Loading.KEY);
-        this.loading = Loading.parse(loading, Loading.LAZY);
-    }
-
-    public String getType() {
-        return this.type;
-    }
-
-    public Settings getSettings() {
-        return this.settings;
-    }
-
-    public Loading getLoading() {
-        return loading;
-    }
-
-    public String getFormat(Settings indexSettings) {
-        String format = settings.get(FORMAT_KEY);
-        if (format == null && indexSettings != null) {
-            format = indexSettings.get(typeFormat);
-        }
-        return format;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-
-        FieldDataType that = (FieldDataType) o;
-
-        if (!settings.equals(that.settings)) return false;
-        if (!type.equals(that.type)) return false;
-
-        return true;
-    }
-
-    @Override
-    public int hashCode() {
-        int result = type.hashCode();
-        result = 31 * result + settings.hashCode();
-        return result;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
index ffa23bf56e4..feacfe59996 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
@@ -20,10 +20,14 @@
 package org.elasticsearch.index.fielddata;
 
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReaderContext;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.ReaderUtil;
 import org.apache.lucene.search.DocIdSet;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.FieldComparatorSource;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.Weight;
@@ -63,18 +67,6 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
                 return null;
             }
         }
-
-        /**
-         * Gets a memory storage hint that should be honored if possible but is not mandatory
-         */
-        public static MemoryStorageFormat getMemoryStorageHint(FieldDataType fieldDataType) {
-            // backwards compatibility
-            String s = fieldDataType.getSettings().get("ordinals");
-            if (s != null) {
-                return "always".equals(s) ? MemoryStorageFormat.ORDINALS : null;
-            }
-            return MemoryStorageFormat.fromString(fieldDataType.getSettings().get(SETTING_MEMORY_STORAGE_HINT));
-        }
     }
 
     /**
@@ -82,11 +74,6 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
      */
     String getFieldName();
 
-    /**
-     * The field data type.
-     */
-    FieldDataType getFieldDataType();
-
     /**
      * Loads the atomic field data for the reader, possibly cached.
      */
@@ -122,11 +109,11 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
         public static class Nested {
 
             private final BitSetProducer rootFilter;
-            private final Weight innerFilter;
+            private final Query innerQuery;
 
-            public Nested(BitSetProducer rootFilter, Weight innerFilter) {
+            public Nested(BitSetProducer rootFilter, Query innerQuery) {
                 this.rootFilter = rootFilter;
-                this.innerFilter = innerFilter;
+                this.innerQuery = innerQuery;
             }
 
             /**
@@ -140,7 +127,10 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
              * Get a {@link DocIdSet} that matches the inner documents.
              */
             public DocIdSetIterator innerDocs(LeafReaderContext ctx) throws IOException {
-                Scorer s = innerFilter.scorer(ctx);
+                final IndexReaderContext topLevelCtx = ReaderUtil.getTopLevelContext(ctx);
+                IndexSearcher indexSearcher = new IndexSearcher(topLevelCtx);
+                Weight weight = indexSearcher.createNormalizedWeight(innerQuery, false);
+                Scorer s = weight.scorer(ctx);
                 return s == null ? null : s.iterator();
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
index 68c05aba3fe..948b19a8afb 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
@@ -48,12 +48,12 @@ public interface IndexFieldDataCache {
         /**
          * Called after the fielddata is loaded during the cache phase
          */
-        default void onCache(ShardId shardId, String fieldName, FieldDataType fieldDataType, Accountable ramUsage){}
+        default void onCache(ShardId shardId, String fieldName, Accountable ramUsage){}
 
         /**
          * Called after the fielddata is unloaded
          */
-        default void onRemoval(ShardId shardId, String fieldName, FieldDataType fieldDataType, boolean wasEvicted, long sizeInBytes){}
+        default void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes){}
     }
 
     class None implements IndexFieldDataCache {
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
index 94e9edc5b94..30e1086e187 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -24,6 +24,7 @@ import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.plain.AbstractGeoPointDVIndexFieldData;
@@ -67,118 +68,21 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Clo
             default:
                 throw new IllegalArgumentException("failed to parse [" + s + "] must be one of [node,node]");
         }
-    }, false, Setting.Scope.INDEX);
+    }, Property.IndexScope);
 
-    private static final IndexFieldData.Builder MISSING_DOC_VALUES_BUILDER = (indexProperties, fieldType, cache, breakerService, mapperService1) -> {
-        throw new IllegalStateException("Can't load fielddata on [" + fieldType.name()
-                + "] of index [" + indexProperties.getIndex().getName() + "] because fielddata is unsupported on fields of type ["
-                + fieldType.fieldDataType().getType() + "]. Use doc values instead.");
-    };
-
-    private static final String ARRAY_FORMAT = "array";
-    private static final String DISABLED_FORMAT = "disabled";
-    private static final String DOC_VALUES_FORMAT = "doc_values";
-    private static final String PAGED_BYTES_FORMAT = "paged_bytes";
-
-    private static final IndexFieldData.Builder DISABLED_BUILDER = new IndexFieldData.Builder() {
-        @Override
-        public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
-                CircuitBreakerService breakerService, MapperService mapperService) {
-            throw new IllegalStateException("Field data loading is forbidden on [" + fieldType.name() + "]");
-        }
-    };
-
-    private final static Map<String, IndexFieldData.Builder> buildersByType;
-    private final static Map<String, IndexFieldData.Builder> docValuesBuildersByType;
-    private final static Map<Tuple<String, String>, IndexFieldData.Builder> buildersByTypeAndFormat;
     private final CircuitBreakerService circuitBreakerService;
 
-    static {
-        Map<String, IndexFieldData.Builder> buildersByTypeBuilder = new HashMap<>();
-        buildersByTypeBuilder.put("string", new PagedBytesIndexFieldData.Builder());
-        buildersByTypeBuilder.put(TextFieldMapper.CONTENT_TYPE, new PagedBytesIndexFieldData.Builder());
-        buildersByTypeBuilder.put(KeywordFieldMapper.CONTENT_TYPE, MISSING_DOC_VALUES_BUILDER);
-        buildersByTypeBuilder.put("float", MISSING_DOC_VALUES_BUILDER);
-        buildersByTypeBuilder.put("double", MISSING_DOC_VALUES_BUILDER);
-        buildersByTypeBuilder.put("byte", MISSING_DOC_VALUES_BUILDER);
-        buildersByTypeBuilder.put("short", MISSING_DOC_VALUES_BUILDER);
-        buildersByTypeBuilder.put("int", MISSING_DOC_VALUES_BUILDER);
-        buildersByTypeBuilder.put("long", MISSING_DOC_VALUES_BUILDER);
-        buildersByTypeBuilder.put("geo_point",  new GeoPointArrayIndexFieldData.Builder());
-        buildersByTypeBuilder.put(ParentFieldMapper.NAME, new ParentChildIndexFieldData.Builder());
-        buildersByTypeBuilder.put(IndexFieldMapper.NAME, new IndexIndexFieldData.Builder());
-        buildersByTypeBuilder.put("binary", DISABLED_BUILDER);
-        buildersByTypeBuilder.put(BooleanFieldMapper.CONTENT_TYPE, MISSING_DOC_VALUES_BUILDER);
-        buildersByType = unmodifiableMap(buildersByTypeBuilder);
-
-
-        docValuesBuildersByType = MapBuilder.<String, IndexFieldData.Builder>newMapBuilder()
-                .put("string", new DocValuesIndexFieldData.Builder())
-                .put(KeywordFieldMapper.CONTENT_TYPE, new DocValuesIndexFieldData.Builder())
-                .put("float", new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.FLOAT))
-                .put("double", new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.DOUBLE))
-                .put("byte", new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.BYTE))
-                .put("short", new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.SHORT))
-                .put("int", new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.INT))
-                .put("long", new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.LONG))
-                .put("geo_point", new AbstractGeoPointDVIndexFieldData.Builder())
-                .put("binary", new BytesBinaryDVIndexFieldData.Builder())
-                .put(BooleanFieldMapper.CONTENT_TYPE, new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.BOOLEAN))
-                .immutableMap();
-
-        buildersByTypeAndFormat = MapBuilder.<Tuple<String, String>, IndexFieldData.Builder>newMapBuilder()
-                .put(Tuple.tuple("string", PAGED_BYTES_FORMAT), new PagedBytesIndexFieldData.Builder())
-                .put(Tuple.tuple("string", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder())
-                .put(Tuple.tuple("string", DISABLED_FORMAT), DISABLED_BUILDER)
-
-                .put(Tuple.tuple(TextFieldMapper.CONTENT_TYPE, PAGED_BYTES_FORMAT), new PagedBytesIndexFieldData.Builder())
-                .put(Tuple.tuple(TextFieldMapper.CONTENT_TYPE, DISABLED_FORMAT), DISABLED_BUILDER)
-
-                .put(Tuple.tuple(KeywordFieldMapper.CONTENT_TYPE, DOC_VALUES_FORMAT),  new DocValuesIndexFieldData.Builder())
-                .put(Tuple.tuple(KeywordFieldMapper.CONTENT_TYPE, DISABLED_FORMAT), DISABLED_BUILDER)
-
-                .put(Tuple.tuple("float", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.FLOAT))
-                .put(Tuple.tuple("float", DISABLED_FORMAT), DISABLED_BUILDER)
-
-                .put(Tuple.tuple("double", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.DOUBLE))
-                .put(Tuple.tuple("double", DISABLED_FORMAT), DISABLED_BUILDER)
-
-                .put(Tuple.tuple("byte", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.BYTE))
-                .put(Tuple.tuple("byte", DISABLED_FORMAT), DISABLED_BUILDER)
-
-                .put(Tuple.tuple("short", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.SHORT))
-                .put(Tuple.tuple("short", DISABLED_FORMAT), DISABLED_BUILDER)
-
-                .put(Tuple.tuple("int", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.INT))
-                .put(Tuple.tuple("int", DISABLED_FORMAT), DISABLED_BUILDER)
-
-                .put(Tuple.tuple("long", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.LONG))
-                .put(Tuple.tuple("long", DISABLED_FORMAT), DISABLED_BUILDER)
-
-                .put(Tuple.tuple("geo_point", ARRAY_FORMAT), new GeoPointArrayIndexFieldData.Builder())
-                .put(Tuple.tuple("geo_point", DOC_VALUES_FORMAT), new AbstractGeoPointDVIndexFieldData.Builder())
-                .put(Tuple.tuple("geo_point", DISABLED_FORMAT), DISABLED_BUILDER)
-
-                .put(Tuple.tuple("binary", DOC_VALUES_FORMAT), new BytesBinaryDVIndexFieldData.Builder())
-                .put(Tuple.tuple("binary", DISABLED_FORMAT), DISABLED_BUILDER)
-
-                .put(Tuple.tuple(BooleanFieldMapper.CONTENT_TYPE, DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.BOOLEAN))
-                .put(Tuple.tuple(BooleanFieldMapper.CONTENT_TYPE, DISABLED_FORMAT), DISABLED_BUILDER)
-
-                .immutableMap();
-    }
-
     private final IndicesFieldDataCache indicesFieldDataCache;
     // the below map needs to be modified under a lock
     private final Map<String, IndexFieldDataCache> fieldDataCaches = new HashMap<>();
     private final MapperService mapperService;
     private static final IndexFieldDataCache.Listener DEFAULT_NOOP_LISTENER = new IndexFieldDataCache.Listener() {
         @Override
-        public void onCache(ShardId shardId, String fieldName, FieldDataType fieldDataType, Accountable ramUsage) {
+        public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) {
         }
 
         @Override
-        public void onRemoval(ShardId shardId, String fieldName, FieldDataType fieldDataType, boolean wasEvicted, long sizeInBytes) {
+        public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) {
         }
     };
     private volatile IndexFieldDataCache.Listener listener = DEFAULT_NOOP_LISTENER;
@@ -222,42 +126,15 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Clo
     @SuppressWarnings("unchecked")
     public <IFD extends IndexFieldData<?>> IFD getForField(MappedFieldType fieldType) {
         final String fieldName = fieldType.name();
-        final FieldDataType type = fieldType.fieldDataType();
-        if (type == null) {
-            throw new IllegalArgumentException("found no fielddata type for field [" + fieldName + "]");
-        }
-        final boolean docValues = fieldType.hasDocValues();
-        IndexFieldData.Builder builder = null;
-        String format = type.getFormat(indexSettings.getSettings());
-        if (format != null && FieldDataType.DOC_VALUES_FORMAT_VALUE.equals(format) && !docValues) {
-            logger.warn("field [" + fieldName + "] has no doc values, will use default field data format");
-            format = null;
-        }
-        if (format != null) {
-            builder = buildersByTypeAndFormat.get(Tuple.tuple(type.getType(), format));
-            if (builder == null) {
-                logger.warn("failed to find format [" + format + "] for field [" + fieldName + "], will use default");
-            }
-        }
-        if (builder == null && docValues) {
-            builder = docValuesBuildersByType.get(type.getType());
-        }
-        if (builder == null) {
-            builder = buildersByType.get(type.getType());
-        }
-        if (builder == null) {
-            throw new IllegalArgumentException("failed to find field data builder for field " + fieldName + ", and type " + type.getType());
-        }
+        IndexFieldData.Builder builder = fieldType.fielddataBuilder();
 
         IndexFieldDataCache cache;
         synchronized (this) {
             cache = fieldDataCaches.get(fieldName);
             if (cache == null) {
-                //  we default to node level cache, which in turn defaults to be unbounded
-                // this means changing the node level settings is simple, just set the bounds there
-                String cacheType = type.getSettings().get("cache", indexSettings.getValue(INDEX_FIELDDATA_CACHE_KEY));
+                String cacheType = indexSettings.getValue(INDEX_FIELDDATA_CACHE_KEY);
                 if (FIELDDATA_CACHE_VALUE_NODE.equals(cacheType)) {
-                    cache = indicesFieldDataCache.buildIndexFieldDataCache(listener, index(), fieldName, type);
+                    cache = indicesFieldDataCache.buildIndexFieldDataCache(listener, index(), fieldName);
                 } else if ("none".equals(cacheType)){
                     cache = new IndexFieldDataCache.None();
                 } else {
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java
index bb31df75348..9e21562e8c7 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java
@@ -51,7 +51,7 @@ public class ShardFieldData implements IndexFieldDataCache.Listener {
     }
 
     @Override
-    public void onCache(ShardId shardId, String fieldName, FieldDataType fieldDataType, Accountable ramUsage) {
+    public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) {
         totalMetric.inc(ramUsage.ramBytesUsed());
         CounterMetric total = perFieldTotals.get(fieldName);
         if (total != null) {
@@ -67,7 +67,7 @@ public class ShardFieldData implements IndexFieldDataCache.Listener {
     }
 
     @Override
-    public void onRemoval(ShardId shardId, String fieldName, FieldDataType fieldDataType, boolean wasEvicted, long sizeInBytes) {
+    public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) {
         if (wasEvicted) {
             evictionsMetric.inc();
         }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java
index dc5041d24ef..e69ba4f4d0b 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java
@@ -70,7 +70,7 @@ public enum GlobalOrdinalsBuilder {
             );
         }
         return new InternalGlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(),
-                indexFieldData.getFieldDataType(), atomicFD, ordinalMap, memorySizeInBytes
+                atomicFD, ordinalMap, memorySizeInBytes
         );
     }
 
@@ -104,7 +104,7 @@ public enum GlobalOrdinalsBuilder {
         }
         final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT);
         return new InternalGlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(),
-                indexFieldData.getFieldDataType(), atomicFD, ordinalMap, 0
+                atomicFD, ordinalMap, 0
         );
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java
index 5e1a2b57401..3e756204002 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java
@@ -25,11 +25,9 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.search.MultiValueMode;
 
 import java.util.Collection;
@@ -41,13 +39,11 @@ import java.util.Collections;
 public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponent implements IndexOrdinalsFieldData, Accountable {
 
     private final String fieldName;
-    private final FieldDataType fieldDataType;
     private final long memorySizeInBytes;
 
-    protected GlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, FieldDataType fieldDataType, long memorySizeInBytes) {
+    protected GlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, long memorySizeInBytes) {
         super(indexSettings);
         this.fieldName = fieldName;
-        this.fieldDataType = fieldDataType;
         this.memorySizeInBytes = memorySizeInBytes;
     }
 
@@ -71,11 +67,6 @@ public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponen
         return fieldName;
     }
 
-    @Override
-    public FieldDataType getFieldDataType() {
-        return fieldDataType;
-    }
-
     @Override
     public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
         throw new UnsupportedOperationException("no global ordinals sorting yet");
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java
index 297c8b0f30c..5b8ef83b10e 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java
@@ -24,9 +24,7 @@ import org.apache.lucene.index.RandomAccessOrds;
 import org.apache.lucene.util.Accountable;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
-import org.elasticsearch.index.mapper.MappedFieldType;
 
 import java.util.Collection;
 
@@ -37,8 +35,8 @@ final class InternalGlobalOrdinalsIndexFieldData extends GlobalOrdinalsIndexFiel
 
     private final Atomic[] atomicReaders;
 
-    InternalGlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, FieldDataType fieldDataType, AtomicOrdinalsFieldData[] segmentAfd, OrdinalMap ordinalMap, long memorySizeInBytes) {
-        super(indexSettings, fieldName, fieldDataType, memorySizeInBytes);
+    InternalGlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, AtomicOrdinalsFieldData[] segmentAfd, OrdinalMap ordinalMap, long memorySizeInBytes) {
+        super(indexSettings, fieldName, memorySizeInBytes);
         this.atomicReaders = new Atomic[segmentAfd.length];
         for (int i = 0; i < segmentAfd.length; i++) {
             atomicReaders[i] = new Atomic(segmentAfd[i], ordinalMap, i);
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java
index 2b69afa5f82..e3dc84a3477 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java
@@ -35,7 +35,6 @@ import org.apache.lucene.util.LongsRef;
 import org.apache.lucene.util.packed.GrowableWriter;
 import org.apache.lucene.util.packed.PackedInts;
 import org.apache.lucene.util.packed.PagedGrowableWriter;
-import org.elasticsearch.common.settings.Settings;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -287,20 +286,13 @@ public final class OrdinalsBuilder implements Closeable {
     private OrdinalsStore ordinals;
     private final LongsRef spare;
 
-    public OrdinalsBuilder(long numTerms, int maxDoc, float acceptableOverheadRatio) throws IOException {
+    public OrdinalsBuilder(int maxDoc, float acceptableOverheadRatio) throws IOException {
         this.maxDoc = maxDoc;
         int startBitsPerValue = 8;
-        if (numTerms >= 0) {
-            startBitsPerValue = PackedInts.bitsRequired(numTerms);
-        }
         ordinals = new OrdinalsStore(maxDoc, startBitsPerValue, acceptableOverheadRatio);
         spare = new LongsRef();
     }
 
-    public OrdinalsBuilder(int maxDoc, float acceptableOverheadRatio) throws IOException {
-        this(-1, maxDoc, acceptableOverheadRatio);
-    }
-
     public OrdinalsBuilder(int maxDoc) throws IOException {
         this(maxDoc, DEFAULT_ACCEPTABLE_OVERHEAD_RATIO);
     }
@@ -413,10 +405,9 @@ public final class OrdinalsBuilder implements Closeable {
     /**
      * Builds an {@link Ordinals} instance from the builders current state.
      */
-    public Ordinals build(Settings settings) {
-        final float acceptableOverheadRatio = settings.getAsFloat("acceptable_overhead_ratio", PackedInts.FASTEST);
-        final boolean forceMultiOrdinals = settings.getAsBoolean(FORCE_MULTI_ORDINALS, false);
-        if (forceMultiOrdinals || numMultiValuedDocs > 0 || MultiOrdinals.significantlySmallerThanSinglePackedOrdinals(maxDoc, numDocsWithValue, getValueCount(), acceptableOverheadRatio)) {
+    public Ordinals build() {
+        final float acceptableOverheadRatio = PackedInts.DEFAULT;
+        if (numMultiValuedDocs > 0 || MultiOrdinals.significantlySmallerThanSinglePackedOrdinals(maxDoc, numDocsWithValue, getValueCount(), acceptableOverheadRatio)) {
             // MultiOrdinals can be smaller than SinglePackedOrdinals for sparse fields
             return new MultiOrdinals(this, acceptableOverheadRatio);
         } else {
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java
index 3d4b6536b6c..23e770121a7 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java
@@ -26,7 +26,6 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
@@ -40,8 +39,8 @@ import java.io.IOException;
 
 public abstract class AbstractGeoPointDVIndexFieldData extends DocValuesIndexFieldData implements IndexGeoPointFieldData {
 
-    AbstractGeoPointDVIndexFieldData(Index index, String fieldName, FieldDataType fieldDataType) {
-        super(index, fieldName, fieldDataType);
+    AbstractGeoPointDVIndexFieldData(Index index, String fieldName) {
+        super(index, fieldName);
     }
 
     @Override
@@ -55,8 +54,8 @@ public abstract class AbstractGeoPointDVIndexFieldData extends DocValuesIndexFie
     public static class GeoPointDVIndexFieldData extends AbstractGeoPointDVIndexFieldData {
         final boolean indexCreatedBefore2x;
 
-        public GeoPointDVIndexFieldData(Index index, String fieldName, FieldDataType fieldDataType, final boolean indexCreatedBefore2x) {
-            super(index, fieldName, fieldDataType);
+        public GeoPointDVIndexFieldData(Index index, String fieldName, final boolean indexCreatedBefore2x) {
+            super(index, fieldName);
             this.indexCreatedBefore2x = indexCreatedBefore2x;
         }
 
@@ -82,8 +81,12 @@ public abstract class AbstractGeoPointDVIndexFieldData extends DocValuesIndexFie
         @Override
         public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
                                        CircuitBreakerService breakerService, MapperService mapperService) {
+            if (indexSettings.getIndexVersionCreated().before(Version.V_2_2_0)
+                    && fieldType.hasDocValues() == false) {
+                return new GeoPointArrayIndexFieldData(indexSettings, fieldType.name(), cache, breakerService);
+            }
             // Ignore breaker
-            return new GeoPointDVIndexFieldData(indexSettings.getIndex(), fieldType.name(), fieldType.fieldDataType(),
+            return new GeoPointDVIndexFieldData(indexSettings.getIndex(), fieldType.name(),
                     indexSettings.getIndexVersionCreated().before(Version.V_2_2_0));
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java
index 151ee92058d..604e866de02 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java
@@ -27,7 +27,6 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AtomicFieldData;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.RamAccountingTermsEnum;
@@ -39,13 +38,11 @@ import java.io.IOException;
 public abstract class AbstractIndexFieldData<FD extends AtomicFieldData> extends AbstractIndexComponent implements IndexFieldData<FD> {
 
     private final String fieldName;
-    protected final FieldDataType fieldDataType;
     protected final IndexFieldDataCache cache;
 
-    public AbstractIndexFieldData(IndexSettings indexSettings, String fieldName, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+    public AbstractIndexFieldData(IndexSettings indexSettings, String fieldName, IndexFieldDataCache cache) {
         super(indexSettings);
         this.fieldName = fieldName;
-        this.fieldDataType = fieldDataType;
         this.cache = cache;
     }
 
@@ -54,11 +51,6 @@ public abstract class AbstractIndexFieldData<FD extends AtomicFieldData> extends
         return this.fieldName;
     }
 
-    @Override
-    public FieldDataType getFieldDataType() {
-        return fieldDataType;
-    }
-
     @Override
     public void clear() {
         cache.clear(fieldName);
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java
index 2c41dece3de..4e0eb218663 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java
@@ -30,7 +30,6 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
@@ -102,8 +101,8 @@ abstract class AbstractIndexGeoPointFieldData extends AbstractIndexFieldData<Ato
         }
     }
 
-    public AbstractIndexGeoPointFieldData(IndexSettings indexSettings, String fieldName, FieldDataType fieldDataType, IndexFieldDataCache cache) {
-        super(indexSettings, fieldName, fieldDataType, cache);
+    public AbstractIndexGeoPointFieldData(IndexSettings indexSettings, String fieldName, IndexFieldDataCache cache) {
+        super(indexSettings, fieldName, cache);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
index 670c6aa47f6..9c71f5bad9b 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
@@ -25,13 +25,10 @@ import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.CharsRefBuilder;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
@@ -41,23 +38,21 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldData<AtomicOrdinalsFieldData> implements IndexOrdinalsFieldData {
 
-    protected Settings frequency;
-    protected Settings regex;
+    private final double minFrequency, maxFrequency;
+    private final int minSegmentSize;
     protected final CircuitBreakerService breakerService;
 
-    protected AbstractIndexOrdinalsFieldData(IndexSettings indexSettings, String fieldName, FieldDataType fieldDataType,
-                                          IndexFieldDataCache cache, CircuitBreakerService breakerService) {
-        super(indexSettings, fieldName, fieldDataType, cache);
-        final Map<String, Settings> groups = fieldDataType.getSettings().getGroups("filter");
-        frequency = groups.get("frequency");
-        regex = groups.get("regex");
+    protected AbstractIndexOrdinalsFieldData(IndexSettings indexSettings, String fieldName,
+            IndexFieldDataCache cache, CircuitBreakerService breakerService,
+            double minFrequency, double maxFrequency, int minSegmentSize) {
+        super(indexSettings, fieldName, cache);
         this.breakerService = breakerService;
+        this.minFrequency = minFrequency;
+        this.maxFrequency = maxFrequency;
+        this.minSegmentSize = minSegmentSize;
     }
 
     @Override
@@ -110,17 +105,24 @@ public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldD
         return AbstractAtomicOrdinalsFieldData.empty();
     }
 
-    protected TermsEnum filter(Terms terms, LeafReader reader) throws IOException {
-        TermsEnum iterator = terms.iterator();
+    protected TermsEnum filter(Terms terms, TermsEnum iterator, LeafReader reader) throws IOException {
         if (iterator == null) {
             return null;
         }
-        if (iterator != null && frequency != null) {
-            iterator = FrequencyFilter.filter(iterator, terms, reader, frequency);
+        int docCount = terms.getDocCount();
+        if (docCount == -1) {
+            docCount = reader.maxDoc();
         }
-
-        if (iterator != null && regex != null) {
-            iterator = RegexFilter.filter(iterator, terms, reader, regex);
+        if (docCount >= minSegmentSize) {
+            final int minFreq = minFrequency > 1.0
+                    ? (int) minFrequency
+                    : (int)(docCount * minFrequency);
+            final int maxFreq = maxFrequency > 1.0
+                    ? (int) maxFrequency
+                    : (int)(docCount * maxFrequency);
+            if (minFreq > 1 || maxFreq < docCount) {
+                iterator = new FrequencyFilter(iterator, minFreq, maxFreq);
+            }
         }
         return iterator;
     }
@@ -135,25 +137,6 @@ public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldD
             this.maxFreq = maxFreq;
         }
 
-        public static TermsEnum filter(TermsEnum toFilter, Terms terms, LeafReader reader, Settings settings) throws IOException {
-            int docCount = terms.getDocCount();
-            if (docCount == -1) {
-                docCount = reader.maxDoc();
-            }
-            final double minFrequency = settings.getAsDouble("min", 0d);
-            final double maxFrequency = settings.getAsDouble("max", docCount+1d);
-            final double minSegmentSize = settings.getAsInt("min_segment_size", 0);
-            if (minSegmentSize < docCount) {
-                final int minFreq = minFrequency > 1.0? (int) minFrequency : (int)(docCount * minFrequency);
-                final int maxFreq = maxFrequency > 1.0? (int) maxFrequency : (int)(docCount * maxFrequency);
-                assert minFreq < maxFreq;
-                return new FrequencyFilter(toFilter, minFreq, maxFreq);
-            }
-
-            return toFilter;
-
-        }
-
         @Override
         protected AcceptStatus accept(BytesRef arg0) throws IOException {
             int docFreq = docFreq();
@@ -164,33 +147,4 @@ public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldD
         }
     }
 
-    private static final class RegexFilter extends FilteredTermsEnum {
-
-        private final Matcher matcher;
-        private final CharsRefBuilder spare = new CharsRefBuilder();
-
-        public RegexFilter(TermsEnum delegate, Matcher matcher) {
-            super(delegate, false);
-            this.matcher = matcher;
-        }
-        public static TermsEnum filter(TermsEnum iterator, Terms terms, LeafReader reader, Settings regex) {
-            String pattern = regex.get("pattern");
-            if (pattern == null) {
-                return iterator;
-            }
-            Pattern p = Pattern.compile(pattern);
-            return new RegexFilter(iterator, p.matcher(""));
-        }
-
-        @Override
-        protected AcceptStatus accept(BytesRef arg0) throws IOException {
-            spare.copyUTF8Bytes(arg0);
-            matcher.reset(spare.get());
-            if (matcher.matches()) {
-                return AcceptStatus.YES;
-            }
-            return AcceptStatus.NO;
-        }
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
index c2a50942566..586ad1f0d48 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.fielddata.plain;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
@@ -29,8 +28,8 @@ import org.elasticsearch.search.MultiValueMode;
 
 public class BinaryDVIndexFieldData extends DocValuesIndexFieldData implements IndexFieldData<BinaryDVAtomicFieldData> {
 
-    public BinaryDVIndexFieldData(Index index, String fieldName, FieldDataType fieldDataType) {
-        super(index, fieldName, fieldDataType);
+    public BinaryDVIndexFieldData(Index index, String fieldName) {
+        super(index, fieldName);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java
index 988ecd61d65..bd3cdd71184 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java
@@ -24,7 +24,6 @@ import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
@@ -37,8 +36,8 @@ import java.io.IOException;
 
 public class BytesBinaryDVIndexFieldData extends DocValuesIndexFieldData implements IndexFieldData<BytesBinaryDVAtomicFieldData> {
 
-    public BytesBinaryDVIndexFieldData(Index index, String fieldName, FieldDataType fieldDataType) {
-        super(index, fieldName, fieldDataType);
+    public BytesBinaryDVIndexFieldData(Index index, String fieldName) {
+        super(index, fieldName);
     }
 
     @Override
@@ -67,7 +66,7 @@ public class BytesBinaryDVIndexFieldData extends DocValuesIndexFieldData impleme
                                        CircuitBreakerService breakerService, MapperService mapperService) {
             // Ignore breaker
             final String fieldName = fieldType.name();
-            return new BytesBinaryDVIndexFieldData(indexSettings.getIndex(), fieldName, fieldType.fieldDataType());
+            return new BytesBinaryDVIndexFieldData(indexSettings.getIndex(), fieldName);
         }
 
     }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java
index 27db531a218..111595859a9 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java
@@ -25,7 +25,6 @@ import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
@@ -46,14 +45,12 @@ public abstract class DocValuesIndexFieldData {
 
     protected final Index index;
     protected final String fieldName;
-    protected final FieldDataType fieldDataType;
     protected final ESLogger logger;
 
-    public DocValuesIndexFieldData(Index index, String fieldName, FieldDataType fieldDataType) {
+    public DocValuesIndexFieldData(Index index, String fieldName) {
         super();
         this.index = index;
         this.fieldName = fieldName;
-        this.fieldDataType = fieldDataType;
         this.logger = Loggers.getLogger(getClass());
     }
 
@@ -61,10 +58,6 @@ public abstract class DocValuesIndexFieldData {
         return fieldName;
     }
 
-    public final FieldDataType getFieldDataType() {
-        return fieldDataType;
-    }
-
     public final void clear() {
         // can't do
     }
@@ -92,19 +85,13 @@ public abstract class DocValuesIndexFieldData {
                                        CircuitBreakerService breakerService, MapperService mapperService) {
             // Ignore Circuit Breaker
             final String fieldName = fieldType.name();
-            final Settings fdSettings = fieldType.fieldDataType().getSettings();
-            final Map<String, Settings> filter = fdSettings.getGroups("filter");
-            if (filter != null && !filter.isEmpty()) {
-                throw new IllegalArgumentException("Doc values field data doesn't support filters [" + fieldName + "]");
-            }
-
             if (BINARY_INDEX_FIELD_NAMES.contains(fieldName)) {
                 assert numericType == null;
-                return new BinaryDVIndexFieldData(indexSettings.getIndex(), fieldName, fieldType.fieldDataType());
+                return new BinaryDVIndexFieldData(indexSettings.getIndex(), fieldName);
             } else if (numericType != null) {
-                return new SortedNumericDVIndexFieldData(indexSettings.getIndex(), fieldName, numericType, fieldType.fieldDataType());
+                return new SortedNumericDVIndexFieldData(indexSettings.getIndex(), fieldName, numericType);
             } else {
-                return new SortedSetDVOrdinalsIndexFieldData(indexSettings, cache, fieldName, breakerService, fieldType.fieldDataType());
+                return new SortedSetDVOrdinalsIndexFieldData(indexSettings, cache, fieldName, breakerService);
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayIndexFieldData.java
index e5724453eee..d484c503c2b 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointArrayIndexFieldData.java
@@ -35,7 +35,6 @@ import org.elasticsearch.common.util.LongArray;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
 import org.elasticsearch.index.fielddata.FieldData;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.ordinals.Ordinals;
@@ -50,18 +49,9 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService;
 public class GeoPointArrayIndexFieldData extends AbstractIndexGeoPointFieldData {
     private final CircuitBreakerService breakerService;
 
-    public static class Builder implements IndexFieldData.Builder {
-        @Override
-        public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
-                                       CircuitBreakerService breakerService, MapperService mapperService) {
-            return new GeoPointArrayIndexFieldData(indexSettings, fieldType.name(), fieldType.fieldDataType(), cache,
-                    breakerService);
-        }
-    }
-
     public GeoPointArrayIndexFieldData(IndexSettings indexSettings, String fieldName,
-                                       FieldDataType fieldDataType, IndexFieldDataCache cache, CircuitBreakerService breakerService) {
-        super(indexSettings, fieldName, fieldDataType, cache);
+                                       IndexFieldDataCache cache, CircuitBreakerService breakerService) {
+        super(indexSettings, fieldName, cache);
         this.breakerService = breakerService;
     }
 
@@ -88,8 +78,7 @@ public class GeoPointArrayIndexFieldData extends AbstractIndexGeoPointFieldData
     private AtomicGeoPointFieldData loadFieldData22(LeafReader reader, NonEstimatingEstimator estimator, Terms terms,
                                                     AtomicGeoPointFieldData data) throws Exception {
         LongArray indexedPoints = BigArrays.NON_RECYCLING_INSTANCE.newLongArray(128);
-        final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_transient_overhead_ratio",
-                OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO);
+        final float acceptableTransientOverheadRatio = OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO;
         boolean success = false;
         try (OrdinalsBuilder builder = new OrdinalsBuilder(reader.maxDoc(), acceptableTransientOverheadRatio)) {
             final TermsEnum termsEnum;
@@ -112,10 +101,9 @@ public class GeoPointArrayIndexFieldData extends AbstractIndexGeoPointFieldData
             }
             indexedPoints = BigArrays.NON_RECYCLING_INSTANCE.resize(indexedPoints, numTerms);
 
-            Ordinals build = builder.build(fieldDataType.getSettings());
+            Ordinals build = builder.build();
             RandomAccessOrds ordinals = build.ordinals();
-            if (!(FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings
-                    .MemoryStorageFormat.ORDINALS)) {
+            if (FieldData.isMultiValued(ordinals) == false) {
                 int maxDoc = reader.maxDoc();
                 LongArray sIndexedPoint = BigArrays.NON_RECYCLING_INSTANCE.newLongArray(reader.maxDoc());
                 for (int i=0; i<maxDoc; ++i) {
@@ -146,9 +134,9 @@ public class GeoPointArrayIndexFieldData extends AbstractIndexGeoPointFieldData
                                                         AtomicGeoPointFieldData data) throws Exception {
         DoubleArray lat = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(128);
         DoubleArray lon = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(128);
-        final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO);
+        final float acceptableTransientOverheadRatio = OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO;
         boolean success = false;
-        try (OrdinalsBuilder builder = new OrdinalsBuilder(terms.size(), reader.maxDoc(), acceptableTransientOverheadRatio)) {
+        try (OrdinalsBuilder builder = new OrdinalsBuilder(reader.maxDoc(), acceptableTransientOverheadRatio)) {
             final GeoPointTermsEnumLegacy iter = new GeoPointTermsEnumLegacy(builder.buildFromTerms(terms.iterator()));
             GeoPoint point;
             long numTerms = 0;
@@ -162,9 +150,9 @@ public class GeoPointArrayIndexFieldData extends AbstractIndexGeoPointFieldData
             lat = BigArrays.NON_RECYCLING_INSTANCE.resize(lat, numTerms);
             lon = BigArrays.NON_RECYCLING_INSTANCE.resize(lon, numTerms);
 
-            Ordinals build = builder.build(fieldDataType.getSettings());
+            Ordinals build = builder.build();
             RandomAccessOrds ordinals = build.ordinals();
-            if (!(FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS)) {
+            if (FieldData.isMultiValued(ordinals) == false) {
                 int maxDoc = reader.maxDoc();
                 DoubleArray sLat = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(reader.maxDoc());
                 DoubleArray sLon = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(reader.maxDoc());
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java
index 7fdacbf53ce..1410e742773 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java
@@ -28,12 +28,12 @@ import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 
 import java.util.Collection;
@@ -101,7 +101,10 @@ public class IndexIndexFieldData extends AbstractIndexOrdinalsFieldData {
     private final AtomicOrdinalsFieldData atomicFieldData;
 
     private IndexIndexFieldData(IndexSettings indexSettings, String name) {
-        super(indexSettings, name, new FieldDataType("string"), null, null);
+        super(indexSettings, name, null, null,
+                TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY,
+                TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY,
+                TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE);
         atomicFieldData = new IndexAtomicFieldData(index().getName());
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java
index ce4f5c55eac..5695f7ef15a 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java
@@ -34,7 +34,6 @@ import org.apache.lucene.util.packed.PackedLongValues;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
@@ -54,16 +53,27 @@ public class PagedBytesIndexFieldData extends AbstractIndexOrdinalsFieldData {
 
     public static class Builder implements IndexFieldData.Builder {
 
+        private final double minFrequency, maxFrequency;
+        private final int minSegmentSize;
+
+        public Builder(double minFrequency, double maxFrequency, int minSegmentSize) {
+            this.minFrequency = minFrequency;
+            this.maxFrequency = maxFrequency;
+            this.minSegmentSize = minSegmentSize;
+        }
+
         @Override
         public IndexOrdinalsFieldData build(IndexSettings indexSettings, MappedFieldType fieldType,
                                                                IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) {
-            return new PagedBytesIndexFieldData(indexSettings, fieldType.name(), fieldType.fieldDataType(), cache, breakerService);
+            return new PagedBytesIndexFieldData(indexSettings, fieldType.name(), cache, breakerService,
+                    minFrequency, maxFrequency, minSegmentSize);
         }
     }
 
     public PagedBytesIndexFieldData(IndexSettings indexSettings, String fieldName,
-                                    FieldDataType fieldDataType, IndexFieldDataCache cache, CircuitBreakerService breakerService) {
-        super(indexSettings, fieldName, fieldDataType, cache, breakerService);
+                                    IndexFieldDataCache cache, CircuitBreakerService breakerService,
+                                    double minFrequency, double maxFrequency, int minSegmentSize) {
+        super(indexSettings, fieldName, cache, breakerService, minFrequency, maxFrequency, minSegmentSize);
     }
 
     @Override
@@ -82,14 +92,7 @@ public class PagedBytesIndexFieldData extends AbstractIndexOrdinalsFieldData {
         final PagedBytes bytes = new PagedBytes(15);
 
         final PackedLongValues.Builder termOrdToBytesOffset = PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
-        final long numTerms;
-        if (regex == null && frequency == null) {
-            numTerms = terms.size();
-        } else {
-            numTerms = -1;
-        }
-        final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat(
-                FilterSettingFields.ACCEPTABLE_TRANSIENT_OVERHEAD_RATIO, OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO);
+        final float acceptableTransientOverheadRatio = OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO;
 
         // Wrap the context in an estimator and use it to either estimate
         // the entire set, or wrap the TermsEnum so it can be calculated
@@ -98,7 +101,7 @@ public class PagedBytesIndexFieldData extends AbstractIndexOrdinalsFieldData {
         TermsEnum termsEnum = estimator.beforeLoad(terms);
         boolean success = false;
 
-        try (OrdinalsBuilder builder = new OrdinalsBuilder(numTerms, reader.maxDoc(), acceptableTransientOverheadRatio)) {
+        try (OrdinalsBuilder builder = new OrdinalsBuilder(reader.maxDoc(), acceptableTransientOverheadRatio)) {
             PostingsEnum docsEnum = null;
             for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) {
                 final long termOrd = builder.nextOrdinal();
@@ -110,7 +113,7 @@ public class PagedBytesIndexFieldData extends AbstractIndexOrdinalsFieldData {
                 }
             }
             PagedBytes.Reader bytesReader = bytes.freeze(true);
-            final Ordinals ordinals = builder.build(fieldDataType.getSettings());
+            final Ordinals ordinals = builder.build();
 
             data = new PagedBytesAtomicFieldData(bytesReader, termOrdToBytesOffset.build(), ordinals);
             success = true;
@@ -199,33 +202,28 @@ public class PagedBytesIndexFieldData extends AbstractIndexOrdinalsFieldData {
          */
         @Override
         public TermsEnum beforeLoad(Terms terms) throws IOException {
-            final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat(
-                    FilterSettingFields.ACCEPTABLE_TRANSIENT_OVERHEAD_RATIO,
-                    OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO);
-
             LeafReader reader = context.reader();
-            // Check if one of the following is present:
-            // - The OrdinalsBuilder overhead has been tweaked away from the default
-            // - A field data filter is present
-            // - A regex filter is present
-            if (acceptableTransientOverheadRatio != OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO ||
-                    fieldDataType.getSettings().getAsDouble(FilterSettingFields.FREQUENCY_MIN, 0d) != 0d ||
-                    fieldDataType.getSettings().getAsDouble(FilterSettingFields.FREQUENCY_MAX, 0d) != 0d ||
-                    fieldDataType.getSettings().getAsDouble(FilterSettingFields.FREQUENCY_MIN_SEGMENT_SIZE, 0d) != 0d ||
-                    fieldDataType.getSettings().get(FilterSettingFields.REGEX_PATTERN) != null) {
+
+            TermsEnum iterator = terms.iterator();
+            TermsEnum filteredIterator = filter(terms, iterator, reader);
+            final boolean filtered = iterator != filteredIterator;
+            iterator = filteredIterator;
+
+            if (filtered) {
                 if (logger.isTraceEnabled()) {
                     logger.trace("Filter exists, can't circuit break normally, using RamAccountingTermsEnum");
                 }
-                return new RamAccountingTermsEnum(filter(terms, reader), breaker, this, this.fieldName);
+                return new RamAccountingTermsEnum(iterator, breaker, this, this.fieldName);
             } else {
                 estimatedBytes = this.estimateStringFieldData();
                 // If we weren't able to estimate, wrap in the RamAccountingTermsEnum
                 if (estimatedBytes == 0) {
-                    return new RamAccountingTermsEnum(filter(terms, reader), breaker, this, this.fieldName);
+                    iterator = new RamAccountingTermsEnum(iterator, breaker, this, this.fieldName);
+                } else {
+                    breaker.addEstimateBytesAndMaybeBreak(estimatedBytes, fieldName);
                 }
 
-                breaker.addEstimateBytesAndMaybeBreak(estimatedBytes, fieldName);
-                return filter(terms, reader);
+                return iterator;
             }
         }
 
@@ -256,12 +254,4 @@ public class PagedBytesIndexFieldData extends AbstractIndexOrdinalsFieldData {
             breaker.addWithoutBreaking(actualUsed);
         }
     }
-
-    static final class FilterSettingFields {
-        static final String ACCEPTABLE_TRANSIENT_OVERHEAD_RATIO = "acceptable_transient_overhead_ratio";
-        static final String FREQUENCY_MIN = "filter.frequency.min";
-        static final String FREQUENCY_MAX = "filter.frequency.max";
-        static final String FREQUENCY_MIN_SEGMENT_SIZE = "filter.frequency.min_segment_size";
-        static final String REGEX_PATTERN = "filter.regex.pattern";
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java
index f7b25832283..d0e2290c0a9 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java
@@ -40,7 +40,6 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AtomicParentChildFieldData;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
@@ -75,9 +74,9 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare
     private final CircuitBreakerService breakerService;
 
     public ParentChildIndexFieldData(IndexSettings indexSettings, String fieldName,
-                                     FieldDataType fieldDataType, IndexFieldDataCache cache, MapperService mapperService,
+                                     IndexFieldDataCache cache, MapperService mapperService,
                                      CircuitBreakerService breakerService) {
-        super(indexSettings, fieldName, fieldDataType, cache);
+        super(indexSettings, fieldName, cache);
         this.breakerService = breakerService;
         Set<String> parentTypes = new HashSet<>();
         for (DocumentMapper mapper : mapperService.docMappers(false)) {
@@ -146,7 +145,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare
                                        MappedFieldType fieldType,
                                        IndexFieldDataCache cache, CircuitBreakerService breakerService,
                                        MapperService mapperService) {
-            return new ParentChildIndexFieldData(indexSettings, fieldType.name(), fieldType.fieldDataType(), cache,
+            return new ParentChildIndexFieldData(indexSettings, fieldType.name(), cache,
                     mapperService, breakerService);
         }
     }
@@ -324,11 +323,6 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare
             return ParentChildIndexFieldData.this.getFieldName();
         }
 
-        @Override
-        public FieldDataType getFieldDataType() {
-            return ParentChildIndexFieldData.this.getFieldDataType();
-        }
-
         @Override
         public AtomicParentChildFieldData load(LeafReaderContext context) {
             assert context.reader().getCoreCacheKey() == leaves.get(context.ord).reader().getCoreCacheKey();
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java
index 3e0dffa89ac..1577225f173 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java
@@ -30,7 +30,6 @@ import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
 import org.elasticsearch.index.fielddata.FieldData;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
@@ -51,8 +50,8 @@ import java.util.Collections;
 public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData implements IndexNumericFieldData {
     private final NumericType numericType;
 
-    public SortedNumericDVIndexFieldData(Index index, String fieldNames, NumericType numericType, FieldDataType fieldDataType) {
-        super(index, fieldNames, fieldDataType);
+    public SortedNumericDVIndexFieldData(Index index, String fieldNames, NumericType numericType) {
+        super(index, fieldNames);
         if (numericType == null) {
             throw new IllegalArgumentException("numericType must be non-null");
         }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
index fc4f6f15d51..87dc1a2eb08 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
@@ -24,7 +24,6 @@ import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
@@ -42,8 +41,8 @@ public class SortedSetDVOrdinalsIndexFieldData extends DocValuesIndexFieldData i
     private final IndexFieldDataCache cache;
     private final CircuitBreakerService breakerService;
 
-    public SortedSetDVOrdinalsIndexFieldData(IndexSettings indexSettings, IndexFieldDataCache cache, String fieldName, CircuitBreakerService breakerService, FieldDataType fieldDataType) {
-        super(indexSettings.getIndex(), fieldName, fieldDataType);
+    public SortedSetDVOrdinalsIndexFieldData(IndexSettings indexSettings, IndexFieldDataCache cache, String fieldName, CircuitBreakerService breakerService) {
+        super(indexSettings.getIndex(), fieldName);
         this.indexSettings = indexSettings;
         this.cache = cache;
         this.breakerService = breakerService;
diff --git a/core/src/main/java/org/elasticsearch/index/fieldstats/FieldStatsProvider.java b/core/src/main/java/org/elasticsearch/index/fieldstats/FieldStatsProvider.java
new file mode 100644
index 00000000000..066d1207e3f
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/fieldstats/FieldStatsProvider.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fieldstats;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.action.fieldstats.FieldStats;
+import org.elasticsearch.action.fieldstats.IndexConstraint;
+import org.elasticsearch.action.fieldstats.IndexConstraint.Comparison;
+import org.elasticsearch.action.fieldstats.IndexConstraint.Property;
+import org.elasticsearch.common.joda.DateMathParser;
+import org.elasticsearch.index.engine.Engine.Searcher;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.core.DateFieldMapper.DateFieldType;
+import org.elasticsearch.index.mapper.ip.IpFieldMapper.IpFieldType;
+import org.joda.time.DateTimeZone;
+
+import java.io.IOException;
+
+/**
+ * Provides a service for gettings the {@link FieldStats} for a given field from
+ * the index.
+ */
+public class FieldStatsProvider {
+
+    private final Searcher searcher;
+    private final MapperService mapperService;
+
+    /**
+     * @param searcher
+     *            the {@link Searcher}to use when retrieving the
+     *            {@link FieldStats}
+     * @param mapperService
+     *            the {@link MapperService}
+     */
+    public FieldStatsProvider(Searcher searcher, MapperService mapperService) {
+        this.searcher = searcher;
+        this.mapperService = mapperService;
+    }
+
+    /**
+     * @param field
+     *            the name of the field to return {@link FieldStats} for.
+     * @return a {@link FieldStats} object for the given field
+     * @throws IOException
+     *             if the field statistics cannot be read
+     */
+    public <T extends Comparable<T>> FieldStats<T> get(String field) throws IOException {
+        MappedFieldType mappedFieldType = mapperService.fullName(field);
+        if (mappedFieldType != null) {
+                IndexReader reader = searcher.reader();
+                Terms terms = MultiFields.getTerms(reader, field);
+                if (terms != null) {
+                    return mappedFieldType.stats(terms, reader.maxDoc());
+                }
+            }
+        return null;
+    }
+
+    /**
+     * @param fieldName
+     *            the fieldName to check
+     * @param from
+     *            the minimum value for the query
+     * @param to
+     *            the maximum value for the query
+     * @param includeLower
+     *            whether the from value is inclusive
+     * @param includeUpper
+     *            whether the to value is inclusive
+     * @param timeZone
+     *            the timeZone to use for date fields
+     * @param dateMathParser
+     *            the {@link DateMathParser} to use for date fields
+     * @return A {@link Relation} indicating the overlap of the range of terms
+     *         for the field with the query range. This method will return:
+     *         <ul>
+     *         <li>{@link Relation#WITHIN} if the range of terms for the field
+     *         in the shard is completely within the query range</li>
+     *         <li>{@link Relation#DISJOINT} if the range of terms for the field
+     *         in the shard is completely outside the query range</li>
+     *         <li>{@link Relation#INTERSECTS} if the range of terms for the
+     *         field in the shard intersects with the query range</li>
+     *         </ul>
+     * @throws IOException
+     *             if the index cannot be read
+     */
+    public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+            DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+        MappedFieldType mappedFieldType = mapperService.fullName(fieldName);
+        FieldStats<?> fieldStats = get(fieldName);
+        if (fieldStats == null) {
+            // No fieldStats for the field so the field doesn't exist on
+            // this shard, so relation is DISJOINT
+            return Relation.DISJOINT;
+        } else {
+            // Convert the from and to values to Strings so they can be used
+            // in the IndexConstraints. Since DateTime is represented as a
+            // Long field in Lucene we need to use the millisecond value of
+            // the DateTime in that case
+            String fromString = null;
+            if (from != null) {
+                if (mappedFieldType instanceof DateFieldType) {
+                    long millis = ((DateFieldType) mappedFieldType).parseToMilliseconds(from, !includeLower, timeZone, dateMathParser);
+                    fromString = fieldStats.stringValueOf(millis, null);
+                } else if (mappedFieldType instanceof IpFieldType) {
+                    if (from instanceof BytesRef) {
+                        from = ((BytesRef) from).utf8ToString();
+                    }
+                    long ipAsLong = ((IpFieldType) mappedFieldType).value(from);
+                    fromString = fieldStats.stringValueOf(ipAsLong, null);
+                } else {
+                    fromString = fieldStats.stringValueOf(from, null);
+                }
+            }
+            String toString = null;
+            if (to != null) {
+                if (mappedFieldType instanceof DateFieldType) {
+                    long millis = ((DateFieldType) mappedFieldType).parseToMilliseconds(to, includeUpper, timeZone, dateMathParser);
+                    toString = fieldStats.stringValueOf(millis, null);
+                } else if (mappedFieldType instanceof IpFieldType) {
+                    if (to instanceof BytesRef) {
+                        to = ((BytesRef) to).utf8ToString();
+                    }
+                    long ipAsLong = ((IpFieldType) mappedFieldType).value(to);
+                    toString = fieldStats.stringValueOf(ipAsLong, null);
+                } else {
+                    toString = fieldStats.stringValueOf(to, null);
+                }
+            }
+            if ((from == null || fieldStats
+                    .match(new IndexConstraint(fieldName, Property.MIN, includeLower ? Comparison.GTE : Comparison.GT, fromString)))
+                    && (to == null || fieldStats.match(
+                            new IndexConstraint(fieldName, Property.MAX, includeUpper ? Comparison.LTE : Comparison.LT, toString)))) {
+                // If the min and max terms for the field are both within
+                // the query range then all documents will match so relation is
+                // WITHIN
+                return Relation.WITHIN;
+            } else if ((to != null && fieldStats
+                    .match(new IndexConstraint(fieldName, Property.MIN, includeUpper ? Comparison.GT : Comparison.GTE, toString)))
+                    || (from != null && fieldStats.match(
+                            new IndexConstraint(fieldName, Property.MAX, includeLower ? Comparison.LT : Comparison.LTE, fromString)))) {
+                // If the min and max terms are both outside the query range
+                // then no document will match so relation is DISJOINT (N.B.
+                // since from <= to we only need
+                // to check one bould for each side of the query range)
+                return Relation.DISJOINT;
+            }
+        }
+        // Range of terms doesn't match any of the constraints so must INTERSECT
+        return Relation.INTERSECTS;
+    }
+
+    /**
+     * An enum used to describe the relation between the range of terms in a
+     * shard when compared with a query range
+     */
+    public static enum Relation {
+        WITHIN, INTERSECTS, DISJOINT;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
index b1f6f7cd9bd..e2c897226c1 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
@@ -25,8 +25,6 @@ import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -140,19 +138,19 @@ public class DocumentMapperParser {
         return docBuilder.build(mapperService);
     }
 
-    public static void checkNoRemainingFields(String fieldName, Map<String, Object> fieldNodeMap, Version indexVersionCreated) {
+    public static void checkNoRemainingFields(String fieldName, Map<?, ?> fieldNodeMap, Version indexVersionCreated) {
         checkNoRemainingFields(fieldNodeMap, indexVersionCreated, "Mapping definition for [" + fieldName + "] has unsupported parameters: ");
     }
 
-    public static void checkNoRemainingFields(Map<String, Object> fieldNodeMap, Version indexVersionCreated, String message) {
+    public static void checkNoRemainingFields(Map<?, ?> fieldNodeMap, Version indexVersionCreated, String message) {
         if (!fieldNodeMap.isEmpty()) {
             throw new MapperParsingException(message + getRemainingFields(fieldNodeMap));
         }
     }
 
-    private static String getRemainingFields(Map<String, ?> map) {
+    private static String getRemainingFields(Map<?, ?> map) {
         StringBuilder remainingFields = new StringBuilder();
-        for (String key : map.keySet()) {
+        for (Object key : map.keySet()) {
             remainingFields.append(" [").append(key).append(" : ").append(map.get(key)).append("]");
         }
         return remainingFields.toString();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index 2694b64783b..0c9f2daa6cb 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -19,8 +19,14 @@
 
 package org.elasticsearch.index.mapper;
 
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
 import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.util.CloseableThreadLocal;
 import org.elasticsearch.common.Strings;
@@ -48,15 +54,8 @@ import org.elasticsearch.index.mapper.object.ArrayValueMapperParser;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
 import org.elasticsearch.index.mapper.object.RootObjectMapper;
 
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
 /** A parser for documents, given mappings from a DocumentMapper */
-class DocumentParser implements Closeable {
+final class DocumentParser implements Closeable {
 
     private CloseableThreadLocal<ParseContext.InternalParseContext> cache = new CloseableThreadLocal<ParseContext.InternalParseContext>() {
         @Override
@@ -99,7 +98,7 @@ class DocumentParser implements Closeable {
 
         reverseOrder(context);
 
-        ParsedDocument doc = parsedDocument(source, context, update(context, mapping));
+        ParsedDocument doc = parsedDocument(source, context, createDynamicUpdate(mapping, docMapper, context.getDynamicMappers()));
         // reset the context to free up memory
         context.reset(null, null, null);
         return doc;
@@ -116,10 +115,7 @@ class DocumentParser implements Closeable {
             // entire type is disabled
             parser.skipChildren();
         } else if (emptyDoc == false) {
-            Mapper update = parseObject(context, mapping.root, true);
-            if (update != null) {
-                context.addDynamicMappingsUpdate(update);
-            }
+            parseObjectOrNested(context, mapping.root, true);
         }
 
         for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) {
@@ -201,11 +197,6 @@ class DocumentParser implements Closeable {
     }
 
 
-    private static Mapping update(ParseContext.InternalParseContext context, Mapping mapping) {
-        Mapper rootDynamicUpdate = context.dynamicMappingsUpdate();
-        return rootDynamicUpdate != null ? mapping.mappingUpdate(rootDynamicUpdate) : null;
-    }
-
     private static MapperParsingException wrapInMapperParsingException(SourceToParse source, Throwable e) {
         // if its already a mapper parsing exception, no need to wrap it...
         if (e instanceof MapperParsingException) {
@@ -220,10 +211,156 @@ class DocumentParser implements Closeable {
         return new MapperParsingException("failed to parse", e);
     }
 
-    static ObjectMapper parseObject(ParseContext context, ObjectMapper mapper, boolean atRoot) throws IOException {
+    /** Creates a Mapping containing any dynamically added fields, or returns null if there were no dynamic mappings. */
+    static Mapping createDynamicUpdate(Mapping mapping, DocumentMapper docMapper, List<Mapper> dynamicMappers) {
+        if (dynamicMappers.isEmpty()) {
+            return null;
+        }
+        // We build a mapping by first sorting the mappers, so that all mappers containing a common prefix
+        // will be processed in a contiguous block. When the prefix is no longer seen, we pop the extra elements
+        // off the stack, merging them upwards into the existing mappers.
+        Collections.sort(dynamicMappers, (Mapper o1, Mapper o2) -> o1.name().compareTo(o2.name()));
+        Iterator<Mapper> dynamicMapperItr = dynamicMappers.iterator();
+        List<ObjectMapper> parentMappers = new ArrayList<>();
+        Mapper firstUpdate = dynamicMapperItr.next();
+        parentMappers.add(createUpdate(mapping.root(), firstUpdate.name().split("\\."), 0, firstUpdate));
+        Mapper previousMapper = null;
+        while (dynamicMapperItr.hasNext()) {
+            Mapper newMapper = dynamicMapperItr.next();
+            if (previousMapper != null && newMapper.name().equals(previousMapper.name())) {
+                // We can see the same mapper more than once, for example, if we had foo.bar and foo.baz, where
+                // foo did not yet exist. This will create 2 copies in dynamic mappings, which should be identical.
+                // Here we just skip over the duplicates, but we merge them to ensure there are no conflicts.
+                newMapper.merge(previousMapper, false);
+                continue;
+            }
+            previousMapper = newMapper;
+            String[] nameParts = newMapper.name().split("\\.");
+
+            // We first need the stack to only contain mappers in common with the previously processed mapper
+            // For example, if the first mapper processed was a.b.c, and we now have a.d, the stack will contain
+            // a.b, and we want to merge b back into the stack so it just contains a
+            int i = removeUncommonMappers(parentMappers, nameParts);
+
+            // Then we need to add back mappers that may already exist within the stack, but are not on it.
+            // For example, if we processed a.b, followed by an object mapper a.c.d, and now are adding a.c.d.e
+            // then the stack will only have a on it because we will have already merged a.c.d into the stack.
+            // So we need to pull a.c, followed by a.c.d, onto the stack so e can be added to the end.
+            i = expandCommonMappers(parentMappers, nameParts, i);
+
+            // If there are still parents of the new mapper which are not on the stack, we need to pull them
+            // from the existing mappings. In order to maintain the invariant that the stack only contains
+            // fields which are updated, we cannot simply add the existing mappers to the stack, since they
+            // may have other subfields which will not be updated. Instead, we pull the mapper from the existing
+            // mappings, and build an update with only the new mapper and its parents. This then becomes our
+            // "new mapper", and can be added to the stack.
+            if (i < nameParts.length - 1) {
+                newMapper = createExistingMapperUpdate(parentMappers, nameParts, i, docMapper, newMapper);
+            }
+
+            if (newMapper instanceof ObjectMapper) {
+                parentMappers.add((ObjectMapper)newMapper);
+            } else {
+                addToLastMapper(parentMappers, newMapper, true);
+            }
+        }
+        popMappers(parentMappers, 1, true);
+        assert parentMappers.size() == 1;
+
+        return mapping.mappingUpdate(parentMappers.get(0));
+    }
+
+    private static void popMappers(List<ObjectMapper> parentMappers, int keepBefore, boolean merge) {
+        assert keepBefore >= 1; // never remove the root mapper
+        // pop off parent mappers not needed by the current mapper,
+        // merging them backwards since they are immutable
+        for (int i = parentMappers.size() - 1; i >= keepBefore; --i) {
+            addToLastMapper(parentMappers, parentMappers.remove(i), merge);
+        }
+    }
+
+    /**
+     * Adds a mapper as an update into the last mapper. If merge is true, the new mapper
+     * will be merged in with other child mappers of the last parent, otherwise it will be a new update.
+     */
+    private static void addToLastMapper(List<ObjectMapper> parentMappers, Mapper mapper, boolean merge) {
+        assert parentMappers.size() >= 1;
+        int lastIndex = parentMappers.size() - 1;
+        ObjectMapper withNewMapper = parentMappers.get(lastIndex).mappingUpdate(mapper);
+        if (merge) {
+            withNewMapper = parentMappers.get(lastIndex).merge(withNewMapper, false);
+        }
+        parentMappers.set(lastIndex, withNewMapper);
+    }
+
+    /**
+     * Removes mappers that exist on the stack, but are not part of the path of the current nameParts,
+     * Returns the next unprocessed index from nameParts.
+     */
+    private static int removeUncommonMappers(List<ObjectMapper> parentMappers, String[] nameParts) {
+        int keepBefore = 1;
+        while (keepBefore < parentMappers.size() &&
+            parentMappers.get(keepBefore).simpleName().equals(nameParts[keepBefore - 1])) {
+            ++keepBefore;
+        }
+        popMappers(parentMappers, keepBefore, true);
+        return keepBefore - 1;
+    }
+
+    /**
+     * Adds mappers from the end of the stack that exist as updates within those mappers.
+     * Returns the next unprocessed index from nameParts.
+     */
+    private static int expandCommonMappers(List<ObjectMapper> parentMappers, String[] nameParts, int i) {
+        ObjectMapper last = parentMappers.get(parentMappers.size() - 1);
+        while (i < nameParts.length - 1 && last.getMapper(nameParts[i]) != null) {
+            Mapper newLast = last.getMapper(nameParts[i]);
+            assert newLast instanceof ObjectMapper;
+            last = (ObjectMapper) newLast;
+            parentMappers.add(last);
+            ++i;
+        }
+        return i;
+    }
+
+    /** Creates an update for intermediate object mappers that are not on the stack, but parents of newMapper. */
+    private static ObjectMapper createExistingMapperUpdate(List<ObjectMapper> parentMappers, String[] nameParts, int i,
+                                                           DocumentMapper docMapper, Mapper newMapper) {
+        String updateParentName = nameParts[i];
+        final ObjectMapper lastParent = parentMappers.get(parentMappers.size() - 1);
+        if (parentMappers.size() > 1) {
+            // only prefix with parent mapper if the parent mapper isn't the root (which has a fake name)
+            updateParentName = lastParent.name() + '.' + nameParts[i];
+        }
+        ObjectMapper updateParent = docMapper.objectMappers().get(updateParentName);
+        assert updateParent != null : updateParentName + " doesn't exist";
+        return createUpdate(updateParent, nameParts, i + 1, newMapper);
+    }
+
+    /** Build an update for the parent which will contain the given mapper and any intermediate fields. */
+    private static ObjectMapper createUpdate(ObjectMapper parent, String[] nameParts, int i, Mapper mapper) {
+        List<ObjectMapper> parentMappers = new ArrayList<>();
+        ObjectMapper previousIntermediate = parent;
+        for (; i < nameParts.length - 1; ++i) {
+            Mapper intermediate = previousIntermediate.getMapper(nameParts[i]);
+            assert intermediate != null : "Field " + previousIntermediate.name() + " does not have a subfield " + nameParts[i];
+            assert intermediate instanceof ObjectMapper;
+            parentMappers.add((ObjectMapper)intermediate);
+            previousIntermediate = (ObjectMapper)intermediate;
+        }
+        if (parentMappers.isEmpty() == false) {
+            // add the new mapper to the stack, and pop down to the original parent level
+            addToLastMapper(parentMappers, mapper, false);
+            popMappers(parentMappers, 1, false);
+            mapper = parentMappers.get(0);
+        }
+        return parent.mappingUpdate(mapper);
+    }
+
+    static void parseObjectOrNested(ParseContext context, ObjectMapper mapper, boolean atRoot) throws IOException {
         if (mapper.isEnabled() == false) {
             context.parser().skipChildren();
-            return null;
+            return;
         }
         XContentParser parser = context.parser();
 
@@ -234,7 +371,7 @@ class DocumentParser implements Closeable {
         XContentParser.Token token = parser.currentToken();
         if (token == XContentParser.Token.VALUE_NULL) {
             // the object is null ("obj1" : null), simply bail
-            return null;
+            return;
         }
 
         if (token.isValue()) {
@@ -256,21 +393,19 @@ class DocumentParser implements Closeable {
         }
 
         ObjectMapper update = null;
-        update = innerParseObject(context, mapper, parser, currentFieldName, token, update);
+        innerParseObject(context, mapper, parser, currentFieldName, token);
         // restore the enable path flag
         if (nested.isNested()) {
             nested(context, nested);
         }
-        return update;
     }
 
-    private static ObjectMapper innerParseObject(ParseContext context, ObjectMapper mapper, XContentParser parser, String currentFieldName, XContentParser.Token token, ObjectMapper update) throws IOException {
+    private static void innerParseObject(ParseContext context, ObjectMapper mapper, XContentParser parser, String currentFieldName, XContentParser.Token token) throws IOException {
         while (token != XContentParser.Token.END_OBJECT) {
-            ObjectMapper newUpdate = null;
             if (token == XContentParser.Token.START_OBJECT) {
-                newUpdate = parseObject(context, mapper, currentFieldName);
+                parseObject(context, mapper, currentFieldName);
             } else if (token == XContentParser.Token.START_ARRAY) {
-                newUpdate = parseArray(context, mapper, currentFieldName);
+                parseArray(context, mapper, currentFieldName);
             } else if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_NULL) {
@@ -278,18 +413,10 @@ class DocumentParser implements Closeable {
             } else if (token == null) {
                 throw new MapperParsingException("object mapping for [" + mapper.name() + "] tried to parse field [" + currentFieldName + "] as object, but got EOF, has a concrete value been provided to it?");
             } else if (token.isValue()) {
-                newUpdate = parseValue(context, mapper, currentFieldName, token);
+                parseValue(context, mapper, currentFieldName, token);
             }
             token = parser.nextToken();
-            if (newUpdate != null) {
-                if (update == null) {
-                    update = newUpdate;
-                } else {
-                    update = update.merge(newUpdate, false);
-                }
-            }
         }
-        return update;
     }
 
     private static void nested(ParseContext context, ObjectMapper.Nested nested) {
@@ -335,33 +462,29 @@ class DocumentParser implements Closeable {
         return context;
     }
 
-    private static Mapper parseObjectOrField(ParseContext context, Mapper mapper) throws IOException {
+    private static void parseObjectOrField(ParseContext context, Mapper mapper) throws IOException {
         if (mapper instanceof ObjectMapper) {
-            return parseObject(context, (ObjectMapper) mapper, false);
+            parseObjectOrNested(context, (ObjectMapper) mapper, false);
         } else {
             FieldMapper fieldMapper = (FieldMapper)mapper;
             Mapper update = fieldMapper.parse(context);
+            if (update != null) {
+                context.addDynamicMapper(update);
+            }
             if (fieldMapper.copyTo() != null) {
                 parseCopyFields(context, fieldMapper, fieldMapper.copyTo().copyToFields());
             }
-            return update;
         }
     }
 
     private static ObjectMapper parseObject(final ParseContext context, ObjectMapper mapper, String currentFieldName) throws IOException {
-        if (currentFieldName == null) {
-            throw new MapperParsingException("object mapping [" + mapper.name() + "] trying to serialize an object with no field associated with it, current value [" + context.parser().textOrNull() + "]");
-        }
+        assert currentFieldName != null;
         context.path().add(currentFieldName);
 
         ObjectMapper update = null;
         Mapper objectMapper = mapper.getMapper(currentFieldName);
         if (objectMapper != null) {
-            final Mapper subUpdate = parseObjectOrField(context, objectMapper);
-            if (subUpdate != null) {
-                // propagate mapping update
-                update = mapper.mappingUpdate(subUpdate);
-            }
+            parseObjectOrField(context, objectMapper);
         } else {
             ObjectMapper.Dynamic dynamic = mapper.dynamic();
             if (dynamic == null) {
@@ -382,8 +505,9 @@ class DocumentParser implements Closeable {
                 }
                 Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path());
                 objectMapper = builder.build(builderContext);
+                context.addDynamicMapper(objectMapper);
                 context.path().add(currentFieldName);
-                update = mapper.mappingUpdate(parseAndMergeUpdate(objectMapper, context));
+                parseObjectOrField(context, objectMapper);
             } else {
                 // not dynamic, read everything up to end object
                 context.parser().skipChildren();
@@ -394,7 +518,7 @@ class DocumentParser implements Closeable {
         return update;
     }
 
-    private static ObjectMapper parseArray(ParseContext context, ObjectMapper parentMapper, String lastFieldName) throws IOException {
+    private static void parseArray(ParseContext context, ObjectMapper parentMapper, String lastFieldName) throws IOException {
         String arrayFieldName = lastFieldName;
         Mapper mapper = parentMapper.getMapper(lastFieldName);
         if (mapper != null) {
@@ -402,15 +526,9 @@ class DocumentParser implements Closeable {
             // expects an array, if so we pass the context straight to the mapper and if not
             // we serialize the array components
             if (mapper instanceof ArrayValueMapperParser) {
-                final Mapper subUpdate = parseObjectOrField(context, mapper);
-                if (subUpdate != null) {
-                    // propagate the mapping update
-                    return parentMapper.mappingUpdate(subUpdate);
-                } else {
-                    return null;
-                }
+                parseObjectOrField(context, mapper);
             } else {
-                return parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
+                parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
             }
         } else {
 
@@ -423,31 +541,34 @@ class DocumentParser implements Closeable {
             } else if (dynamic == ObjectMapper.Dynamic.TRUE) {
                 Mapper.Builder builder = context.root().findTemplateBuilder(context, arrayFieldName, "object");
                 if (builder == null) {
-                    return parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
+                    parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
+                    return;
                 }
                 Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path());
                 mapper = builder.build(builderContext);
-                if (mapper != null && mapper instanceof ArrayValueMapperParser) {
+                assert mapper != null;
+                if (mapper instanceof ArrayValueMapperParser) {
+                    context.addDynamicMapper(mapper);
                     context.path().add(arrayFieldName);
-                    mapper = parseAndMergeUpdate(mapper, context);
-                    return parentMapper.mappingUpdate(mapper);
+                    parseObjectOrField(context, mapper);
                 } else {
-                    return parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
+                    parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
                 }
             } else {
-                return parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
+                // TODO: shouldn't this skip, not parse?
+                parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
             }
         }
     }
 
-    private static ObjectMapper parseNonDynamicArray(ParseContext context, ObjectMapper mapper, String lastFieldName, String arrayFieldName) throws IOException {
+    private static void parseNonDynamicArray(ParseContext context, ObjectMapper mapper, String lastFieldName, String arrayFieldName) throws IOException {
         XContentParser parser = context.parser();
         XContentParser.Token token;
         while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
             if (token == XContentParser.Token.START_OBJECT) {
-                return parseObject(context, mapper, lastFieldName);
+                parseObject(context, mapper, lastFieldName);
             } else if (token == XContentParser.Token.START_ARRAY) {
-                return parseArray(context, mapper, lastFieldName);
+                parseArray(context, mapper, lastFieldName);
             } else if (token == XContentParser.Token.FIELD_NAME) {
                 lastFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_NULL) {
@@ -455,25 +576,20 @@ class DocumentParser implements Closeable {
             } else if (token == null) {
                 throw new MapperParsingException("object mapping for [" + mapper.name() + "] with array for [" + arrayFieldName + "] tried to parse as array, but got EOF, is there a mismatch in types for the same field?");
             } else {
-                return parseValue(context, mapper, lastFieldName, token);
+                parseValue(context, mapper, lastFieldName, token);
             }
         }
-        return null;
     }
 
-    private static ObjectMapper parseValue(final ParseContext context, ObjectMapper parentMapper, String currentFieldName, XContentParser.Token token) throws IOException {
+    private static void parseValue(final ParseContext context, ObjectMapper parentMapper, String currentFieldName, XContentParser.Token token) throws IOException {
         if (currentFieldName == null) {
             throw new MapperParsingException("object mapping [" + parentMapper.name() + "] trying to serialize a value with no field associated with it, current value [" + context.parser().textOrNull() + "]");
         }
         Mapper mapper = parentMapper.getMapper(currentFieldName);
         if (mapper != null) {
-            Mapper subUpdate = parseObjectOrField(context, mapper);
-            if (subUpdate == null) {
-                return null;
-            }
-            return parentMapper.mappingUpdate(subUpdate);
+            parseObjectOrField(context, mapper);
         } else {
-            return parseDynamicValue(context, parentMapper, currentFieldName, token);
+            parseDynamicValue(context, parentMapper, currentFieldName, token);
         }
     }
 
@@ -589,7 +705,7 @@ class DocumentParser implements Closeable {
                     Double.parseDouble(text);
                     Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double");
                     if (builder == null) {
-                        builder = new DoubleFieldMapper.Builder(currentFieldName);
+                        builder = new FloatFieldMapper.Builder(currentFieldName);
                     }
                     return builder;
                 } catch (NumberFormatException e) {
@@ -641,7 +757,7 @@ class DocumentParser implements Closeable {
         throw new IllegalStateException("Can't handle serializing a dynamic type with content token [" + token + "] and field name [" + currentFieldName + "]");
     }
 
-    private static ObjectMapper parseDynamicValue(final ParseContext context, ObjectMapper parentMapper, String currentFieldName, XContentParser.Token token) throws IOException {
+    private static void parseDynamicValue(final ParseContext context, ObjectMapper parentMapper, String currentFieldName, XContentParser.Token token) throws IOException {
         ObjectMapper.Dynamic dynamic = parentMapper.dynamic();
         if (dynamic == null) {
             dynamic = dynamicOrDefault(context.root().dynamic());
@@ -650,7 +766,7 @@ class DocumentParser implements Closeable {
             throw new StrictDynamicMappingException(parentMapper.fullPath(), currentFieldName);
         }
         if (dynamic == ObjectMapper.Dynamic.FALSE) {
-            return null;
+            return;
         }
         final String path = context.path().pathAsText(currentFieldName);
         final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path());
@@ -668,14 +784,9 @@ class DocumentParser implements Closeable {
             // try to not introduce a conflict
             mapper = mapper.updateFieldType(Collections.singletonMap(path, existingFieldType));
         }
+        context.addDynamicMapper(mapper);
 
-        mapper = parseAndMergeUpdate(mapper, context);
-
-        ObjectMapper update = null;
-        if (mapper != null) {
-            update = parentMapper.mappingUpdate(mapper);
-        }
-        return update;
+        parseObjectOrField(context, mapper);
     }
 
     /** Creates instances of the fields that the current field should be copied to */
@@ -713,8 +824,9 @@ class DocumentParser implements Closeable {
             // The path of the dest field might be completely different from the current one so we need to reset it
             context = context.overridePath(new ContentPath(0));
 
-            String[] paths = Strings.splitStringToArray(field, '.');
-            String fieldName = paths[paths.length-1];
+            // TODO: why Strings.splitStringToArray instead of String.split?
+            final String[] paths = Strings.splitStringToArray(field, '.');
+            final String fieldName = paths[paths.length-1];
             ObjectMapper mapper = context.root();
             ObjectMapper[] mappers = new ObjectMapper[paths.length-1];
             if (paths.length > 1) {
@@ -745,6 +857,7 @@ class DocumentParser implements Closeable {
                                 if (mapper.nested() != ObjectMapper.Nested.NO) {
                                     throw new MapperParsingException("It is forbidden to create dynamic nested objects ([" + context.path().pathAsText(paths[i]) + "]) through `copy_to`");
                                 }
+                                context.addDynamicMapper(mapper);
                                 break;
                             case FALSE:
                               // Maybe we should log something to tell the user that the copy_to is ignored in this case.
@@ -759,36 +872,10 @@ class DocumentParser implements Closeable {
                     parent = mapper;
                 }
             }
-            ObjectMapper update = parseDynamicValue(context, mapper, fieldName, context.parser().currentToken());
-            assert update != null; // we are parsing a dynamic value so we necessarily created a new mapping
-
-            if (paths.length > 1) {
-                for (int i = paths.length - 2; i >= 0; i--) {
-                    ObjectMapper parent = context.root();
-                    if (i > 0) {
-                        parent = mappers[i-1];
-                    }
-                    assert parent != null;
-                    update = parent.mappingUpdate(update);
-                }
-            }
-            context.addDynamicMappingsUpdate(update);
+            parseDynamicValue(context, mapper, fieldName, context.parser().currentToken());
         }
     }
 
-    /**
-     * Parse the given {@code context} with the given {@code mapper} and apply
-     * the potential mapping update in-place. This method is useful when
-     * composing mapping updates.
-     */
-    private static <M extends Mapper> M parseAndMergeUpdate(M mapper, ParseContext context) throws IOException {
-        final Mapper update = parseObjectOrField(context, mapper);
-        if (update != null) {
-            mapper = (M) mapper.merge(update, false);
-        }
-        return mapper;
-    }
-
     private static ObjectMapper.Dynamic dynamicOrDefault(ObjectMapper.Dynamic dynamic) {
         return dynamic == null ? ObjectMapper.Dynamic.TRUE : dynamic;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
index dba1355a395..8077a732142 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
@@ -25,14 +25,13 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
 import org.elasticsearch.Version;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.core.TypeParsers;
 import org.elasticsearch.index.mapper.internal.AllFieldMapper;
 import org.elasticsearch.index.similarity.SimilarityProvider;
@@ -49,8 +48,10 @@ import java.util.Map;
 import java.util.stream.StreamSupport;
 
 public abstract class FieldMapper extends Mapper implements Cloneable {
-    public static final Setting<Boolean> IGNORE_MALFORMED_SETTING = Setting.boolSetting("index.mapping.ignore_malformed", false, false, Setting.Scope.INDEX);
-    public static final Setting<Boolean> COERCE_SETTING = Setting.boolSetting("index.mapping.coerce", false, false, Setting.Scope.INDEX);
+    public static final Setting<Boolean> IGNORE_MALFORMED_SETTING =
+        Setting.boolSetting("index.mapping.ignore_malformed", false, Property.IndexScope);
+    public static final Setting<Boolean> COERCE_SETTING =
+        Setting.boolSetting("index.mapping.coerce", false, Property.IndexScope);
     public abstract static class Builder<T extends Builder, Y extends FieldMapper> extends Mapper.Builder<T, Y> {
 
         protected final MappedFieldType fieldType;
@@ -61,8 +62,6 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
         protected Boolean includeInAll;
         protected boolean indexOptionsSet = false;
         protected boolean docValuesSet = false;
-        @Nullable
-        protected Settings fieldDataSettings;
         protected final MultiFields.Builder multiFieldsBuilder;
         protected CopyTo copyTo;
 
@@ -200,16 +199,6 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
             return builder;
         }
 
-        public T normsLoading(MappedFieldType.Loading normsLoading) {
-            this.fieldType.setNormsLoading(normsLoading);
-            return builder;
-        }
-
-        public T fieldDataSettings(Settings settings) {
-            this.fieldDataSettings = settings;
-            return builder;
-        }
-
         public Builder nullValue(Object nullValue) {
             this.fieldType.setNullValue(nullValue);
             return this;
@@ -230,7 +219,7 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
         }
 
         protected boolean defaultDocValues(Version indexCreated) {
-            if (indexCreated.onOrAfter(Version.V_5_0_0)) {
+            if (indexCreated.onOrAfter(Version.V_5_0_0_alpha1)) {
                 // add doc values by default to keyword (boolean, numerics, etc.) fields
                 return fieldType.tokenized() == false;
             } else {
@@ -240,14 +229,13 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
 
         protected void setupFieldType(BuilderContext context) {
             fieldType.setName(buildFullName(context));
+            if (context.indexCreatedVersion().before(Version.V_5_0_0_alpha1)) {
+                fieldType.setOmitNorms(fieldType.omitNorms() && fieldType.boost() == 1.0f);
+            }
             if (fieldType.indexAnalyzer() == null && fieldType.tokenized() == false && fieldType.indexOptions() != IndexOptions.NONE) {
                 fieldType.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
                 fieldType.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
             }
-            if (fieldDataSettings != null) {
-                Settings settings = Settings.builder().put(fieldType.fieldDataType().getSettings()).put(fieldDataSettings).build();
-                fieldType.setFieldDataType(new FieldDataType(fieldType.fieldDataType().getType(), settings));
-            }
             boolean defaultDocValues = defaultDocValues(context.indexCreatedVersion());
             defaultFieldType.setHasDocValues(defaultDocValues);
             if (docValuesSet == false) {
@@ -301,7 +289,7 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
                 if (!customBoost()
                         // don't set boosts eg. on dv fields
                         && field.fieldType().indexOptions() != IndexOptions.NONE
-                        && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+                        && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                     field.setBoost(fieldType().boost());
                 }
                 context.doc().add(field);
@@ -374,7 +362,8 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
             // this can happen if this mapper represents a mapping update
             return this;
         } else if (fieldType.getClass() != newFieldType.getClass()) {
-            throw new IllegalStateException("Mixing up field types: " + fieldType.getClass() + " != " + newFieldType.getClass());
+            throw new IllegalStateException("Mixing up field types: " +
+                fieldType.getClass() + " != " + newFieldType.getClass() + " on field " + fieldType.name());
         }
         MultiFields updatedMultiFields = multiFields.updateFieldType(fullNameToFieldType);
         if (fieldType == newFieldType && multiFields == updatedMultiFields) {
@@ -415,19 +404,15 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
         if (includeDefaults || fieldType().storeTermVectors() != defaultFieldType.storeTermVectors()) {
             builder.field("term_vector", termVectorOptionsToString(fieldType()));
         }
-        if (includeDefaults || fieldType().omitNorms() != defaultFieldType.omitNorms() || fieldType().normsLoading() != null) {
-            builder.startObject("norms");
-            if (includeDefaults || fieldType().omitNorms() != defaultFieldType.omitNorms()) {
-                builder.field("enabled", !fieldType().omitNorms());
-            }
-            if (fieldType().normsLoading() != null) {
-                builder.field(MappedFieldType.Loading.KEY, fieldType().normsLoading());
-            }
-            builder.endObject();
+        if (includeDefaults || fieldType().omitNorms() != defaultFieldType.omitNorms()) {
+            builder.field("norms", fieldType().omitNorms() == false);
         }
         if (indexed && (includeDefaults || fieldType().indexOptions() != defaultFieldType.indexOptions())) {
             builder.field("index_options", indexOptionToString(fieldType().indexOptions()));
         }
+        if (includeDefaults || fieldType().eagerGlobalOrdinals() != defaultFieldType.eagerGlobalOrdinals()) {
+            builder.field("eager_global_ordinals", fieldType().eagerGlobalOrdinals());
+        }
 
         if (fieldType().similarity() != null) {
             builder.field("similarity", fieldType().similarity().name());
@@ -435,9 +420,6 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
             builder.field("similarity", SimilarityService.DEFAULT_SIMILARITY);
         }
 
-        if (includeDefaults || hasCustomFieldDataSettings()) {
-            builder.field("fielddata", fieldType().fieldDataType().getSettings().getAsMap());
-        }
         multiFields.toXContent(builder, params);
 
         if (copyTo != null) {
@@ -517,10 +499,6 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
         return indexed;
     }
 
-    protected boolean hasCustomFieldDataSettings() {
-        return fieldType().fieldDataType() != null && fieldType().fieldDataType().equals(defaultFieldType.fieldDataType()) == false;
-    }
-
     protected abstract String contentType();
 
     public static class MultiFields {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
index 5e9378e2f55..5f6fddf09ef 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
@@ -154,8 +154,6 @@ class FieldTypeLookup implements Iterable<MappedFieldType> {
         for (MappedFieldType fieldType : this) {
             if (Regex.simpleMatch(pattern, fieldType.name())) {
                 fields.add(fieldType.name());
-            } else if (Regex.simpleMatch(pattern, fieldType.name())) {
-                fields.add(fieldType.name());
             }
         }
         return fields;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
index 10b165ff4c5..9eafa8ccd71 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
@@ -37,11 +37,10 @@ import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.index.fielddata.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.similarity.SimilarityProvider;
@@ -55,46 +54,6 @@ import java.util.Objects;
  */
 public abstract class MappedFieldType extends FieldType {
 
-    public enum Loading {
-        LAZY {
-            @Override
-            public String toString() {
-                return LAZY_VALUE;
-            }
-        },
-        EAGER {
-            @Override
-            public String toString() {
-                return EAGER_VALUE;
-            }
-        },
-        EAGER_GLOBAL_ORDINALS {
-            @Override
-            public String toString() {
-                return EAGER_GLOBAL_ORDINALS_VALUE;
-            }
-        };
-
-        public static final String KEY = "loading";
-        public static final String EAGER_GLOBAL_ORDINALS_VALUE = "eager_global_ordinals";
-        public static final String EAGER_VALUE = "eager";
-        public static final String LAZY_VALUE = "lazy";
-
-        public static Loading parse(String loading, Loading defaultValue) {
-            if (Strings.isNullOrEmpty(loading)) {
-                return defaultValue;
-            } else if (EAGER_GLOBAL_ORDINALS_VALUE.equalsIgnoreCase(loading)) {
-                return EAGER_GLOBAL_ORDINALS;
-            } else if (EAGER_VALUE.equalsIgnoreCase(loading)) {
-                return EAGER;
-            } else if (LAZY_VALUE.equalsIgnoreCase(loading)) {
-                return LAZY;
-            } else {
-                throw new MapperParsingException("Unknown [" + KEY + "] value: [" + loading + "]");
-            }
-        }
-    }
-
     private String name;
     private float boost;
     // TODO: remove this docvalues flag and use docValuesType
@@ -103,10 +62,9 @@ public abstract class MappedFieldType extends FieldType {
     private NamedAnalyzer searchAnalyzer;
     private NamedAnalyzer searchQuoteAnalyzer;
     private SimilarityProvider similarity;
-    private Loading normsLoading;
-    private FieldDataType fieldDataType;
     private Object nullValue;
     private String nullValueAsString; // for sending null value to _all field
+    private boolean eagerGlobalOrdinals;
 
     protected MappedFieldType(MappedFieldType ref) {
         super(ref);
@@ -117,10 +75,9 @@ public abstract class MappedFieldType extends FieldType {
         this.searchAnalyzer = ref.searchAnalyzer();
         this.searchQuoteAnalyzer = ref.searchQuoteAnalyzer();
         this.similarity = ref.similarity();
-        this.normsLoading = ref.normsLoading();
-        this.fieldDataType = ref.fieldDataType();
         this.nullValue = ref.nullValue();
         this.nullValueAsString = ref.nullValueAsString();
+        this.eagerGlobalOrdinals = ref.eagerGlobalOrdinals;
     }
 
     public MappedFieldType() {
@@ -130,12 +87,16 @@ public abstract class MappedFieldType extends FieldType {
         setOmitNorms(false);
         setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
         setBoost(1.0f);
-        fieldDataType = new FieldDataType(typeName());
     }
 
     @Override
     public abstract MappedFieldType clone();
 
+    /** Return a fielddata builder for this field. */
+    public IndexFieldData.Builder fielddataBuilder() {
+        throw new IllegalArgumentException("Fielddata is not supported on fields of type [" + typeName() + "]");
+    }
+
     @Override
     public boolean equals(Object o) {
         if (!super.equals(o)) return false;
@@ -158,8 +119,7 @@ public abstract class MappedFieldType extends FieldType {
             Objects.equals(indexAnalyzer, fieldType.indexAnalyzer) &&
             Objects.equals(searchAnalyzer, fieldType.searchAnalyzer) &&
             Objects.equals(searchQuoteAnalyzer(), fieldType.searchQuoteAnalyzer()) &&
-            Objects.equals(normsLoading, fieldType.normsLoading) &&
-            Objects.equals(fieldDataType, fieldType.fieldDataType) &&
+            Objects.equals(eagerGlobalOrdinals, fieldType.eagerGlobalOrdinals) &&
             Objects.equals(nullValue, fieldType.nullValue) &&
             Objects.equals(nullValueAsString, fieldType.nullValueAsString);
     }
@@ -167,7 +127,7 @@ public abstract class MappedFieldType extends FieldType {
     @Override
     public int hashCode() {
         return Objects.hash(super.hashCode(), name, boost, docValues, indexAnalyzer, searchAnalyzer, searchQuoteAnalyzer,
-            similarity == null ? null : similarity.name(), normsLoading, fieldDataType, nullValue, nullValueAsString);
+            eagerGlobalOrdinals, similarity == null ? null : similarity.name(), nullValue, nullValueAsString);
     }
 
     // norelease: we need to override freeze() and add safety checks that all settings are actually set
@@ -205,7 +165,7 @@ public abstract class MappedFieldType extends FieldType {
             conflicts.add("mapper [" + name() + "] has different [doc_values] values");
         }
         if (omitNorms() && !other.omitNorms()) {
-            conflicts.add("mapper [" + name() + "] has different [omit_norms] values, cannot change from disable to enabled");
+            conflicts.add("mapper [" + name() + "] has different [norms] values, cannot change from disable to enabled");
         }
         if (storeTermVectors() != other.storeTermVectors()) {
             conflicts.add("mapper [" + name() + "] has different [store_term_vector] values");
@@ -242,21 +202,18 @@ public abstract class MappedFieldType extends FieldType {
             if (boost() != other.boost()) {
                 conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [boost] across all types.");
             }
-            if (normsLoading() != other.normsLoading()) {
-                conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [norms.loading] across all types.");
-            }
             if (Objects.equals(searchAnalyzer(), other.searchAnalyzer()) == false) {
                 conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [search_analyzer] across all types.");
             }
             if (Objects.equals(searchQuoteAnalyzer(), other.searchQuoteAnalyzer()) == false) {
                 conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [search_quote_analyzer] across all types.");
             }
-            if (Objects.equals(fieldDataType(), other.fieldDataType()) == false) {
-                conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [fielddata] across all types.");
-            }
             if (Objects.equals(nullValue(), other.nullValue()) == false) {
                 conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [null_value] across all types.");
             }
+            if (eagerGlobalOrdinals() != other.eagerGlobalOrdinals()) {
+                conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [eager_global_ordinals] across all types.");
+            }
         }
     }
 
@@ -286,15 +243,6 @@ public abstract class MappedFieldType extends FieldType {
         this.boost = boost;
     }
 
-    public FieldDataType fieldDataType() {
-        return fieldDataType;
-    }
-
-    public void setFieldDataType(FieldDataType fieldDataType) {
-        checkIfFrozen();
-        this.fieldDataType = fieldDataType;
-    }
-
     public boolean hasDocValues() {
         return docValues;
     }
@@ -304,15 +252,6 @@ public abstract class MappedFieldType extends FieldType {
         this.docValues = hasDocValues;
     }
 
-    public Loading normsLoading() {
-        return normsLoading;
-    }
-
-    public void setNormsLoading(Loading normsLoading) {
-        checkIfFrozen();
-        this.normsLoading = normsLoading;
-    }
-
     public NamedAnalyzer indexAnalyzer() {
         return indexAnalyzer;
     }
@@ -402,7 +341,7 @@ public abstract class MappedFieldType extends FieldType {
     public Query termQuery(Object value, @Nullable QueryShardContext context) {
         TermQuery query = new TermQuery(createTerm(value));
         if (boost == 1f ||
-            (context != null && context.indexVersionCreated().before(Version.V_5_0_0))) {
+            (context != null && context.indexVersionCreated().before(Version.V_5_0_0_alpha1))) {
             return query;
         }
         return new BoostQuery(query, boost);
@@ -468,4 +407,21 @@ public abstract class MappedFieldType extends FieldType {
     public Query queryStringTermQuery(Term term) {
         return null;
     }
+
+    protected final void failIfNoDocValues() {
+        if (hasDocValues() == false) {
+            throw new IllegalStateException("Can't load fielddata on [" + name()
+                + "] because fielddata is unsupported on fields of type ["
+                + typeName() + "]. Use doc values instead.");
+        }
+    }
+
+    public boolean eagerGlobalOrdinals() {
+        return eagerGlobalOrdinals;
+    }
+
+    public void setEagerGlobalOrdinals(boolean eagerGlobalOrdinals) {
+        checkIfFrozen();
+        this.eagerGlobalOrdinals = eagerGlobalOrdinals;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java
index 4dd43db0517..6a9a402a5ff 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java
@@ -76,6 +76,7 @@ public abstract class Mapper implements ToXContent, Iterable<Mapper> {
             return this.name;
         }
 
+        /** Returns a newly built mapper. */
         public abstract Y build(BuilderContext context);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index b25f5f6a02d..eaf897e7fbd 100755
--- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -27,17 +27,18 @@ import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.mapper.Mapper.BuilderContext;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
+import org.elasticsearch.index.percolator.PercolatorFieldMapper;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.InvalidTypeNameException;
 import org.elasticsearch.indices.TypeMissingException;
 import org.elasticsearch.indices.mapper.MapperRegistry;
-import org.elasticsearch.percolator.PercolatorService;
 import org.elasticsearch.script.ScriptService;
 
 import java.io.Closeable;
@@ -81,9 +82,11 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
     }
 
     public static final String DEFAULT_MAPPING = "_default_";
-    public static final Setting<Long> INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING = Setting.longSetting("index.mapping.nested_fields.limit", 50L, 0, true, Setting.Scope.INDEX);
+    public static final Setting<Long> INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING =
+        Setting.longSetting("index.mapping.nested_fields.limit", 50L, 0, Property.Dynamic, Property.IndexScope);
     public static final boolean INDEX_MAPPER_DYNAMIC_DEFAULT = true;
-    public static final Setting<Boolean> INDEX_MAPPER_DYNAMIC_SETTING = Setting.boolSetting("index.mapper.dynamic", INDEX_MAPPER_DYNAMIC_DEFAULT, false, Setting.Scope.INDEX);
+    public static final Setting<Boolean> INDEX_MAPPER_DYNAMIC_SETTING =
+        Setting.boolSetting("index.mapper.dynamic", INDEX_MAPPER_DYNAMIC_DEFAULT, Property.IndexScope);
     private static ObjectHashSet<String> META_FIELDS = ObjectHashSet.from(
             "_uid", "_id", "_type", "_all", "_parent", "_routing", "_index",
             "_size", "_timestamp", "_ttl"
@@ -330,7 +333,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
     }
 
     private boolean typeNameStartsWithIllegalDot(DocumentMapper mapper) {
-        return mapper.type().startsWith(".") && !PercolatorService.TYPE_NAME.equals(mapper.type());
+        return mapper.type().startsWith(".") && !PercolatorFieldMapper.TYPE_NAME.equals(mapper.type());
     }
 
     private boolean assertSerialization(DocumentMapper mapper) {
@@ -402,7 +405,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
 
     public DocumentMapper parse(String mappingType, CompressedXContent mappingSource, boolean applyDefault) throws MapperParsingException {
         String defaultMappingSource;
-        if (PercolatorService.TYPE_NAME.equals(mappingType)) {
+        if (PercolatorFieldMapper.TYPE_NAME.equals(mappingType)) {
             defaultMappingSource = this.defaultPercolatorMappingSource;
         }  else {
             defaultMappingSource = this.defaultMappingSource;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
index a003fe153c2..da1ae91c9b5 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
@@ -331,13 +331,13 @@ public abstract class ParseContext {
         }
 
         @Override
-        public void addDynamicMappingsUpdate(Mapper update) {
-            in.addDynamicMappingsUpdate(update);
+        public void addDynamicMapper(Mapper update) {
+            in.addDynamicMapper(update);
         }
 
         @Override
-        public Mapper dynamicMappingsUpdate() {
-            return in.dynamicMappingsUpdate();
+        public List<Mapper> getDynamicMappers() {
+            return in.getDynamicMappers();
         }
     }
 
@@ -369,7 +369,7 @@ public abstract class ParseContext {
 
         private AllEntries allEntries = new AllEntries();
 
-        private Mapper dynamicMappingsUpdate = null;
+        private List<Mapper> dynamicMappers = new ArrayList<>();
 
         public InternalParseContext(@Nullable Settings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper, ContentPath path) {
             this.indexSettings = indexSettings;
@@ -394,7 +394,7 @@ public abstract class ParseContext {
             this.source = source == null ? null : sourceToParse.source();
             this.path.reset();
             this.allEntries = new AllEntries();
-            this.dynamicMappingsUpdate = null;
+            this.dynamicMappers = new ArrayList<>();
         }
 
         @Override
@@ -536,18 +536,13 @@ public abstract class ParseContext {
         }
 
         @Override
-        public void addDynamicMappingsUpdate(Mapper mapper) {
-            assert mapper instanceof RootObjectMapper : mapper;
-            if (dynamicMappingsUpdate == null) {
-                dynamicMappingsUpdate = mapper;
-            } else {
-                dynamicMappingsUpdate = dynamicMappingsUpdate.merge(mapper, false);
-            }
+        public void addDynamicMapper(Mapper mapper) {
+            dynamicMappers.add(mapper);
         }
 
         @Override
-        public Mapper dynamicMappingsUpdate() {
-            return dynamicMappingsUpdate;
+        public List<Mapper> getDynamicMappers() {
+            return dynamicMappers;
         }
     }
 
@@ -747,12 +742,12 @@ public abstract class ParseContext {
     public abstract StringBuilder stringBuilder();
 
     /**
-     * Add a dynamic update to the root object mapper.
+     * Add a new mapper dynamically created while parsing.
      */
-    public abstract void addDynamicMappingsUpdate(Mapper update);
+    public abstract void addDynamicMapper(Mapper update);
 
     /**
-     * Get dynamic updates to the root object mapper.
+     * Get dynamic mappers created while parsing.
      */
-    public abstract Mapper dynamicMappingsUpdate();
+    public abstract List<Mapper> getDynamicMappers();
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
index 9fc12a3dd0c..4eb8b3569d4 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
@@ -32,6 +32,8 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.plain.BytesBinaryDVIndexFieldData;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
@@ -131,6 +133,12 @@ public class BinaryFieldMapper extends FieldMapper {
         public Object valueForSearch(Object value) {
             return value(value);
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            failIfNoDocValues();
+            return new BytesBinaryDVIndexFieldData.Builder();
+        }
     }
 
     protected BinaryFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
index 4b49d644d5f..02d6a536812 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
@@ -29,6 +29,9 @@ import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
@@ -107,8 +110,6 @@ public class BooleanFieldMapper extends FieldMapper {
                     }
                     builder.nullValue(lenientNodeBooleanValue(propNode));
                     iterator.remove();
-                } else if (parseMultiField(builder, name, parserContext, propName, propNode)) {
-                    iterator.remove();
                 }
             }
             return builder;
@@ -191,6 +192,12 @@ public class BooleanFieldMapper extends FieldMapper {
         public boolean useTermQueryWithQueryString() {
             return true;
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            failIfNoDocValues();
+            return new DocValuesIndexFieldData.Builder().numericType(NumericType.BOOLEAN);
+        }
     }
 
     protected BooleanFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
index 86818a3999e..c42f0b7cd42 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
@@ -38,6 +38,9 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericIntegerAnalyzer;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -185,6 +188,12 @@ public class ByteFieldMapper extends NumberFieldMapper {
                 maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue
             );
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            failIfNoDocValues();
+            return new DocValuesIndexFieldData.Builder().numericType(NumericType.BYTE);
+        }
     }
 
     protected ByteFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
@@ -252,7 +261,7 @@ public class ByteFieldMapper extends NumberFieldMapper {
                     context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost);
                 }
             } else if (parser.currentToken() == XContentParser.Token.START_OBJECT
-                    && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+                    && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                 XContentParser.Token token;
                 String currentFieldName = null;
                 Byte objValue = fieldType().nullValue();
@@ -285,7 +294,7 @@ public class ByteFieldMapper extends NumberFieldMapper {
         }
         if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
             CustomByteNumericField field = new CustomByteNumericField(value, fieldType());
-            if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+            if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                 field.setBoost(boost);
             }
             fields.add(field);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java
index 057a8957121..c0b9173cea0 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java
@@ -183,7 +183,6 @@ public class CompletionFieldMapper extends FieldMapper implements ArrayValueMapp
         private ContextMappings contextMappings = null;
 
         public CompletionFieldType() {
-            setFieldDataType(null);
         }
 
         private CompletionFieldType(CompletionFieldType ref) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
index 048c58297cb..5d66fb62c76 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
@@ -44,7 +44,9 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericDateAnalyzer;
-import org.elasticsearch.index.fielddata.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -252,7 +254,6 @@ public class DateFieldMapper extends NumberFieldMapper {
 
         public DateFieldType() {
             super(LegacyNumericType.LONG);
-            setFieldDataType(new FieldDataType("long"));
         }
 
         protected DateFieldType(DateFieldType ref) {
@@ -417,10 +418,15 @@ public class DateFieldMapper extends NumberFieldMapper {
         }
 
         public long parseToMilliseconds(Object value, boolean inclusive, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) {
+            if (value instanceof Long) {
+                return ((Long) value).longValue();
+            }
+
             DateMathParser dateParser = dateMathParser();
             if (forcedDateParser != null) {
                 dateParser = forcedDateParser;
             }
+
             String strValue;
             if (value instanceof BytesRef) {
                 strValue = ((BytesRef) value).utf8ToString();
@@ -429,6 +435,12 @@ public class DateFieldMapper extends NumberFieldMapper {
             }
             return dateParser.parse(strValue, now(), inclusive, zone);
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            failIfNoDocValues();
+            return new DocValuesIndexFieldData.Builder().numericType(NumericType.LONG);
+        }
     }
 
     protected DateFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Explicit<Boolean> ignoreMalformed,Explicit<Boolean> coerce,
@@ -476,7 +488,7 @@ public class DateFieldMapper extends NumberFieldMapper {
             } else if (token == XContentParser.Token.VALUE_NUMBER) {
                 dateAsString = parser.text();
             } else if (token == XContentParser.Token.START_OBJECT
-                    && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+                    && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                 String currentFieldName = null;
                 while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                     if (token == XContentParser.Token.FIELD_NAME) {
@@ -511,7 +523,7 @@ public class DateFieldMapper extends NumberFieldMapper {
         if (value != null) {
             if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
                 CustomLongNumericField field = new CustomLongNumericField(value, fieldType());
-                if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+                if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                     field.setBoost(boost);
                 }
                 fields.add(field);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
index e7550dc1f92..69d56439b33 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
@@ -40,6 +40,9 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericDoubleAnalyzer;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -188,6 +191,12 @@ public class DoubleFieldMapper extends NumberFieldMapper {
                 maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue
             );
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            failIfNoDocValues();
+            return new DocValuesIndexFieldData.Builder().numericType(NumericType.DOUBLE);
+        }
     }
 
     protected DoubleFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Explicit<Boolean> ignoreMalformed,
@@ -244,7 +253,7 @@ public class DoubleFieldMapper extends NumberFieldMapper {
                     context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost);
                 }
             } else if (parser.currentToken() == XContentParser.Token.START_OBJECT
-                    && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+                    && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                 XContentParser.Token token;
                 String currentFieldName = null;
                 Double objValue = fieldType().nullValue();
@@ -278,7 +287,7 @@ public class DoubleFieldMapper extends NumberFieldMapper {
 
         if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
             CustomDoubleNumericField field = new CustomDoubleNumericField(value, fieldType());
-            if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+            if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                 field.setBoost(boost);
             }
             fields.add(field);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
index 93cf3a7cfaf..48b9ca95875 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
@@ -41,6 +41,9 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericFloatAnalyzer;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -189,6 +192,12 @@ public class FloatFieldMapper extends NumberFieldMapper {
                 maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue
             );
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            failIfNoDocValues();
+            return new DocValuesIndexFieldData.Builder().numericType(NumericType.FLOAT);
+        }
     }
 
     protected FloatFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
@@ -256,7 +265,7 @@ public class FloatFieldMapper extends NumberFieldMapper {
                     context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost);
                 }
             } else if (parser.currentToken() == XContentParser.Token.START_OBJECT
-                    && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+                    && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                 XContentParser.Token token;
                 String currentFieldName = null;
                 Float objValue = fieldType().nullValue();
@@ -290,7 +299,7 @@ public class FloatFieldMapper extends NumberFieldMapper {
 
         if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
             CustomFloatNumericField field = new CustomFloatNumericField(value, fieldType());
-            if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+            if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                 field.setBoost(boost);
             }
             fields.add(field);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
index fa7191cafbf..860377d0879 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
@@ -40,6 +40,9 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericIntegerAnalyzer;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -194,6 +197,12 @@ public class IntegerFieldMapper extends NumberFieldMapper {
                 maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue
             );
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            failIfNoDocValues();
+            return new DocValuesIndexFieldData.Builder().numericType(NumericType.INT);
+        }
     }
 
     protected IntegerFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
@@ -261,7 +270,7 @@ public class IntegerFieldMapper extends NumberFieldMapper {
                     context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost);
                 }
             } else if (parser.currentToken() == XContentParser.Token.START_OBJECT
-                    && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+                    && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                 XContentParser.Token token;
                 String currentFieldName = null;
                 Integer objValue = fieldType().nullValue();
@@ -298,7 +307,7 @@ public class IntegerFieldMapper extends NumberFieldMapper {
     protected void addIntegerFields(ParseContext context, List<Field> fields, int value, float boost) {
         if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
             CustomIntegerNumericField field = new CustomIntegerNumericField(value, fieldType());
-            if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+            if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                 field.setBoost(boost);
             }
             fields.add(field);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java
index 3f01493590c..ab36b738c83 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java
@@ -29,6 +29,9 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
@@ -92,12 +95,9 @@ public final class KeywordFieldMapper extends FieldMapper implements AllFieldMap
             return super.indexOptions(indexOptions);
         }
 
-        @Override
-        protected void setupFieldType(BuilderContext context) {
-            if (!omitNormsSet && fieldType.boost() != 1.0f) {
-                fieldType.setOmitNorms(false);
-            }
-            super.setupFieldType(context);
+        public Builder eagerGlobalOrdinals(boolean eagerGlobalOrdinals) {
+            fieldType().setEagerGlobalOrdinals(eagerGlobalOrdinals);
+            return builder;
         }
 
         @Override
@@ -128,7 +128,11 @@ public final class KeywordFieldMapper extends FieldMapper implements AllFieldMap
                 } else if (propName.equals("ignore_above")) {
                     builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
                     iterator.remove();
-                } else if (parseMultiField(builder, name, parserContext, propName, propNode)) {
+                } else if (propName.equals("norms")) {
+                    builder.omitNorms(XContentMapValues.nodeBooleanValue(propNode) == false);
+                    iterator.remove();
+                } else if (propName.equals("eager_global_ordinals")) {
+                    builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode));
                     iterator.remove();
                 }
             }
@@ -168,6 +172,12 @@ public final class KeywordFieldMapper extends FieldMapper implements AllFieldMap
             }
             return termQuery(nullValue(), null);
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            failIfNoDocValues();
+            return new DocValuesIndexFieldData.Builder();
+        }
     }
 
     private Boolean includeInAll;
@@ -180,6 +190,13 @@ public final class KeywordFieldMapper extends FieldMapper implements AllFieldMap
         this.ignoreAbove = ignoreAbove;
     }
 
+    /** Values that have more chars than the return value of this method will
+     *  be skipped at parsing time. */
+    // pkg-private for testing
+    int ignoreAbove() {
+        return ignoreAbove;
+    }
+
     @Override
     protected KeywordFieldMapper clone() {
         return (KeywordFieldMapper) super.clone();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
index a1acf0ab58a..b261df7a221 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
@@ -40,6 +40,9 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericLongAnalyzer;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -192,6 +195,12 @@ public class LongFieldMapper extends NumberFieldMapper {
                 maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue
             );
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            failIfNoDocValues();
+            return new DocValuesIndexFieldData.Builder().numericType(NumericType.LONG);
+        }
     }
 
     protected LongFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
@@ -249,7 +258,7 @@ public class LongFieldMapper extends NumberFieldMapper {
                     context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost);
                 }
             } else if (parser.currentToken() == XContentParser.Token.START_OBJECT
-                    && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+                    && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                 XContentParser.Token token;
                 String currentFieldName = null;
                 Long objValue = fieldType().nullValue();
@@ -282,7 +291,7 @@ public class LongFieldMapper extends NumberFieldMapper {
         }
         if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
             CustomLongNumericField field = new CustomLongNumericField(value, fieldType());
-            if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+            if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                 field.setBoost(boost);
             }
             fields.add(field);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
index 7c2a38eaee7..b73dfc59930 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
@@ -31,12 +31,17 @@ import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.IndexableFieldType;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
@@ -52,7 +57,9 @@ import java.util.List;
  *
  */
 public abstract class NumberFieldMapper extends FieldMapper implements AllFieldMapper.IncludeInAll {
-    private static final Setting<Boolean> COERCE_SETTING = Setting.boolSetting("index.mapping.coerce", true, false, Setting.Scope.INDEX); // this is private since it has a different default
+    // this is private since it has a different default
+    private static final Setting<Boolean> COERCE_SETTING =
+        Setting.boolSetting("index.mapping.coerce", true, Property.IndexScope);
 
     public static class Defaults {
 
@@ -113,7 +120,6 @@ public abstract class NumberFieldMapper extends FieldMapper implements AllFieldM
 
         protected void setupFieldType(BuilderContext context) {
             super.setupFieldType(context);
-            fieldType.setOmitNorms(fieldType.omitNorms() && fieldType.boost() == 1.0f);
             int precisionStep = fieldType.numericPrecisionStep();
             if (precisionStep <= 0 || precisionStep >= maxPrecisionStep()) {
                 fieldType.setNumericPrecisionStep(Integer.MAX_VALUE);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
index 56b1e9a78f2..eb00e0744c8 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
@@ -40,6 +40,9 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericIntegerAnalyzer;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -190,6 +193,12 @@ public class ShortFieldMapper extends NumberFieldMapper {
                 maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue
             );
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            failIfNoDocValues();
+            return new DocValuesIndexFieldData.Builder().numericType(NumericType.SHORT);
+        }
     }
 
     protected ShortFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
@@ -257,7 +266,7 @@ public class ShortFieldMapper extends NumberFieldMapper {
                     context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost);
                 }
             } else if (parser.currentToken() == XContentParser.Token.START_OBJECT
-                    && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+                    && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                 XContentParser.Token token;
                 String currentFieldName = null;
                 Short objValue = fieldType().nullValue();
@@ -290,7 +299,7 @@ public class ShortFieldMapper extends NumberFieldMapper {
         }
         if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
             CustomShortNumericField field = new CustomShortNumericField(value, fieldType());
-            if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+            if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                 field.setBoost(boost);
             }
             fields.add(field);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
index c4659a6571e..c87efcd9218 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
@@ -26,11 +26,18 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
+import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
@@ -39,9 +46,13 @@ import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.internal.AllFieldMapper;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
 
 import static org.apache.lucene.index.IndexOptions.NONE;
 import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField;
@@ -52,7 +63,20 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
     public static final String CONTENT_TYPE = "string";
     private static final int POSITION_INCREMENT_GAP_USE_ANALYZER = -1;
 
+    // If a string field is created on 5.x and all parameters are in this list then we
+    // will automatically upgrade to a text/keyword field. Otherwise we will just fail
+    // saying that string fields are not supported anymore.
+    private static final Set<String> SUPPORTED_PARAMETERS_FOR_AUTO_UPGRADE = new HashSet<>(Arrays.asList(
+            "type",
+            // most common parameters, for which the upgrade is straightforward
+            "index", "store", "doc_values", "omit_norms", "norms", "fields", "copy_to",
+            "fielddata", "ignore_above"));
+
     public static class Defaults {
+        public static double FIELDDATA_MIN_FREQUENCY = 0;
+        public static double FIELDDATA_MAX_FREQUENCY = Integer.MAX_VALUE;
+        public static int FIELDDATA_MIN_SEGMENT_SIZE = 0;
+
         public static final MappedFieldType FIELD_TYPE = new StringFieldType();
 
         static {
@@ -83,6 +107,11 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
             builder = this;
         }
 
+        @Override
+        public StringFieldType fieldType() {
+            return (StringFieldType) super.fieldType();
+        }
+
         @Override
         public Builder searchAnalyzer(NamedAnalyzer searchAnalyzer) {
             super.searchAnalyzer(searchAnalyzer);
@@ -99,6 +128,31 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
             return this;
         }
 
+        public Builder fielddata(boolean fielddata) {
+            fieldType().setFielddata(fielddata);
+            return builder;
+        }
+
+        public Builder eagerGlobalOrdinals(boolean eagerGlobalOrdinals) {
+            fieldType().setEagerGlobalOrdinals(eagerGlobalOrdinals);
+            return builder;
+        }
+
+        public Builder fielddataFrequencyFilter(double minFreq, double maxFreq, int minSegmentSize) {
+            fieldType().setFielddataMinFrequency(minFreq);
+            fieldType().setFielddataMaxFrequency(maxFreq);
+            fieldType().setFielddataMinSegmentSize(minSegmentSize);
+            return builder;
+        }
+
+        @Override
+        protected void setupFieldType(BuilderContext context) {
+            super.setupFieldType(context);
+            if (fieldType().hasDocValues() && ((StringFieldType) fieldType()).fielddata()) {
+                ((StringFieldType) fieldType()).setFielddata(false);
+            }
+        }
+
         @Override
         public StringFieldMapper build(BuilderContext context) {
             if (positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) {
@@ -123,20 +177,80 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
             }
             setupFieldType(context);
             StringFieldMapper fieldMapper = new StringFieldMapper(
-                    name, fieldType, defaultFieldType, positionIncrementGap, ignoreAbove,
+                    name, fieldType(), defaultFieldType, positionIncrementGap, ignoreAbove,
                     context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
             return fieldMapper.includeInAll(includeInAll);
         }
     }
 
     public static class TypeParser implements Mapper.TypeParser {
+        private final DeprecationLogger deprecationLogger;
+
+        public TypeParser() {
+            ESLogger logger = Loggers.getLogger(getClass());
+            this.deprecationLogger = new DeprecationLogger(logger);
+        }
+
         @Override
         public Mapper.Builder parse(String fieldName, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
-            // TODO: temporarily disabled to give Kibana time to upgrade to text/keyword mappings
-            /*if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0)) {
+            if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha1)) {
+                // Automatically upgrade simple mappings for ease of upgrade, otherwise fail
+                if (SUPPORTED_PARAMETERS_FOR_AUTO_UPGRADE.containsAll(node.keySet())) {
+                    deprecationLogger.deprecated("The [string] field is deprecated, please use [text] or [keyword] instead on [{}]",
+                            fieldName);
+                    final Object index = node.remove("index");
+                    final boolean keyword = index != null && "analyzed".equals(index) == false;
+                    {
+                        // upgrade the index setting
+                        node.put("index", "no".equals(index) == false);
+                    }
+                    {
+                        // upgrade norms settings
+                        Object norms = node.remove("norms");
+                        if (norms instanceof Map) {
+                            norms = ((Map<?,?>) norms).get("enabled");
+                        }
+                        if (norms != null) {
+                            node.put("norms", TypeParsers.nodeBooleanValue("norms", norms, parserContext));
+                        }
+                        Object omitNorms = node.remove("omit_norms");
+                        if (omitNorms != null) {
+                            node.put("norms", TypeParsers.nodeBooleanValue("omit_norms", omitNorms, parserContext) == false);
+                        }
+                    }
+                    {
+                        // upgrade fielddata settings
+                        Object fielddataO = node.get("fielddata");
+                        if (fielddataO instanceof Map) {
+                            Map<?,?> fielddata = (Map<?, ?>) fielddataO;
+                            if (keyword == false) {
+                                node.put("fielddata", "disabled".equals(fielddata.get("format")) == false);
+                                Map<?,?> fielddataFilter = (Map<?, ?>) fielddata.get("filter");
+                                if (fielddataFilter != null) {
+                                    Map<?,?> frequencyFilter = (Map<?, ?>) fielddataFilter.get("frequency");
+                                    frequencyFilter.keySet().retainAll(Arrays.asList("min", "max", "min_segment_size"));
+                                    node.put("fielddata_frequency_filter", frequencyFilter);
+                                }
+                            } else {
+                                node.remove("fielddata");
+                            }
+                            final Object loading = fielddata.get("loading");
+                            if (loading != null) {
+                                node.put("eager_global_ordinals", "eager_global_ordinals".equals(loading));
+                            }
+                        }
+                    }
+                    if (keyword) {
+                        return new KeywordFieldMapper.TypeParser().parse(fieldName, node, parserContext);
+                    } else {
+                        return new TextFieldMapper.TypeParser().parse(fieldName, node, parserContext);
+                    }
+
+                }
                 throw new IllegalArgumentException("The [string] type is removed in 5.0. You should now use either a [text] "
                         + "or [keyword] field instead for field [" + fieldName + "]");
-            }*/
+            }
+            
             StringFieldMapper.Builder builder = new StringFieldMapper.Builder(fieldName);
             // hack for the fact that string can't just accept true/false for
             // the index property and still accepts no/not_analyzed/analyzed
@@ -159,6 +273,21 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                     throw new IllegalArgumentException("Can't parse [index] value [" + index + "] for field [" + fieldName + "], expected [true], [false], [no], [not_analyzed] or [analyzed]");
                 }
             }
+            final Object fielddataObject = node.get("fielddata");
+            if (fielddataObject instanceof Map) {
+                Map<?,?> fielddata = (Map<?, ?>) fielddataObject;
+                final Object loading = fielddata.get("loading");
+                if (loading != null) {
+                    node.put("eager_global_ordinals", "eager_global_ordinals".equals(loading));
+                }
+                Map<?,?> fielddataFilter = (Map<?, ?>) fielddata.get("filter");
+                if (fielddataFilter != null) {
+                    Map<?,?> frequencyFilter = (Map<?, ?>) fielddataFilter.get("frequency");
+                    frequencyFilter.keySet().retainAll(Arrays.asList("min", "max", "min_segment_size"));
+                    node.put("fielddata_frequency_filter", frequencyFilter);
+                }
+                node.put("fielddata", "disabled".equals(fielddata.get("format")) == false);
+            }
             parseTextField(builder, fieldName, node, parserContext);
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
@@ -191,7 +320,19 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                 } else if (propName.equals("ignore_above")) {
                     builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
                     iterator.remove();
-                } else if (parseMultiField(builder, fieldName, parserContext, propName, propNode)) {
+                } else if (propName.equals("fielddata")) {
+                    builder.fielddata(XContentMapValues.nodeBooleanValue(propNode));
+                    iterator.remove();
+                } else if (propName.equals("eager_global_ordinals")) {
+                    builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode));
+                    iterator.remove();
+                } else if (propName.equals("fielddata_frequency_filter")) {
+                    Map<?,?> frequencyFilter = (Map<?, ?>) propNode;
+                    double minFrequency = XContentMapValues.nodeDoubleValue(frequencyFilter.remove("min"), 0);
+                    double maxFrequency = XContentMapValues.nodeDoubleValue(frequencyFilter.remove("max"), Integer.MAX_VALUE);
+                    int minSegmentSize = XContentMapValues.nodeIntegerValue(frequencyFilter.remove("min_segment_size"), 0);
+                    builder.fielddataFrequencyFilter(minFrequency, maxFrequency, minSegmentSize);
+                    DocumentMapperParser.checkNoRemainingFields(propName, frequencyFilter, parserContext.indexVersionCreated());
                     iterator.remove();
                 }
             }
@@ -201,10 +342,42 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
 
     public static final class StringFieldType extends MappedFieldType {
 
-        public StringFieldType() {}
+        private boolean fielddata;
+        private double fielddataMinFrequency;
+        private double fielddataMaxFrequency;
+        private int fielddataMinSegmentSize;
+
+        public StringFieldType() {
+            fielddata = true;
+            fielddataMinFrequency = Defaults.FIELDDATA_MIN_FREQUENCY;
+            fielddataMaxFrequency = Defaults.FIELDDATA_MAX_FREQUENCY;
+            fielddataMinSegmentSize = Defaults.FIELDDATA_MIN_SEGMENT_SIZE;
+        }
 
         protected StringFieldType(StringFieldType ref) {
             super(ref);
+            this.fielddata = ref.fielddata;
+            this.fielddataMinFrequency = ref.fielddataMinFrequency;
+            this.fielddataMaxFrequency = ref.fielddataMaxFrequency;
+            this.fielddataMinSegmentSize = ref.fielddataMinSegmentSize;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (super.equals(o) == false) {
+                return false;
+            }
+            StringFieldType that = (StringFieldType) o;
+            return fielddata == that.fielddata
+                    && fielddataMinFrequency == that.fielddataMinFrequency
+                    && fielddataMaxFrequency == that.fielddataMaxFrequency
+                    && fielddataMinSegmentSize == that.fielddataMinSegmentSize;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(super.hashCode(), fielddata,
+                    fielddataMinFrequency, fielddataMaxFrequency, fielddataMinSegmentSize);
         }
 
         public StringFieldType clone() {
@@ -216,6 +389,67 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
             return CONTENT_TYPE;
         }
 
+        @Override
+        public void checkCompatibility(MappedFieldType other,
+                List<String> conflicts, boolean strict) {
+            super.checkCompatibility(other, conflicts, strict);
+            StringFieldType otherType = (StringFieldType) other;
+            if (strict) {
+                if (fielddata() != otherType.fielddata()) {
+                    conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [fielddata] "
+                            + "across all types.");
+                }
+                if (fielddataMinFrequency() != otherType.fielddataMinFrequency()) {
+                    conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update "
+                            + "[fielddata_frequency_filter.min] across all types.");
+                }
+                if (fielddataMaxFrequency() != otherType.fielddataMaxFrequency()) {
+                    conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update "
+                            + "[fielddata_frequency_filter.max] across all types.");
+                }
+                if (fielddataMinSegmentSize() != otherType.fielddataMinSegmentSize()) {
+                    conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update "
+                            + "[fielddata_frequency_filter.min_segment_size] across all types.");
+                }
+            }
+        }
+
+        public boolean fielddata() {
+            return fielddata;
+        }
+
+        public void setFielddata(boolean fielddata) {
+            checkIfFrozen();
+            this.fielddata = fielddata;
+        }
+
+        public double fielddataMinFrequency() {
+            return fielddataMinFrequency;
+        }
+
+        public void setFielddataMinFrequency(double fielddataMinFrequency) {
+            checkIfFrozen();
+            this.fielddataMinFrequency = fielddataMinFrequency;
+        }
+
+        public double fielddataMaxFrequency() {
+            return fielddataMaxFrequency;
+        }
+
+        public void setFielddataMaxFrequency(double fielddataMaxFrequency) {
+            checkIfFrozen();
+            this.fielddataMaxFrequency = fielddataMaxFrequency;
+        }
+
+        public int fielddataMinSegmentSize() {
+            return fielddataMinSegmentSize;
+        }
+
+        public void setFielddataMinSegmentSize(int fielddataMinSegmentSize) {
+            checkIfFrozen();
+            this.fielddataMinSegmentSize = fielddataMinSegmentSize;
+        }
+
         @Override
         public String value(Object value) {
             if (value == null) {
@@ -231,24 +465,42 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
             }
             return termQuery(nullValue(), null);
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            if (hasDocValues()) {
+                return new DocValuesIndexFieldData.Builder();
+            } else if (fielddata) {
+                return new PagedBytesIndexFieldData.Builder(fielddataMinFrequency, fielddataMaxFrequency, fielddataMinSegmentSize);
+            } else {
+                throw new IllegalStateException("Fielddata is disabled on analyzed string fields by default. Set fielddata=true on ["
+                        + name() + "] in order to load fielddata in memory by uninverting the inverted index. Note that this can however "
+                        + "use significant memory.");
+            }
+        }
     }
 
     private Boolean includeInAll;
     private int positionIncrementGap;
     private int ignoreAbove;
 
-    protected StringFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
+    protected StringFieldMapper(String simpleName, StringFieldType fieldType, MappedFieldType defaultFieldType,
                                 int positionIncrementGap, int ignoreAbove,
                                 Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
         super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
-        // TODO: temporarily disabled to give Kibana time to upgrade to text/keyword mappings
-        /*if (Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0)) {
+        if (Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0_alpha1)) {
             throw new IllegalArgumentException("The [string] type is removed in 5.0. You should now use either a [text] "
                     + "or [keyword] field instead for field [" + fieldType.name() + "]");
-        }*/
+        }
         if (fieldType.tokenized() && fieldType.indexOptions() != NONE && fieldType().hasDocValues()) {
             throw new MapperParsingException("Field [" + fieldType.name() + "] cannot be analyzed and have doc values");
         }
+        if (fieldType.hasDocValues() && (
+                fieldType.fielddataMinFrequency() != Defaults.FIELDDATA_MIN_FREQUENCY
+                || fieldType.fielddataMaxFrequency() != Defaults.FIELDDATA_MAX_FREQUENCY
+                || fieldType.fielddataMinSegmentSize() != Defaults.FIELDDATA_MIN_SEGMENT_SIZE)) {
+            throw new MapperParsingException("Field [" + fieldType.name() + "] cannot have doc values and use fielddata filtering");
+        }
         this.positionIncrementGap = positionIncrementGap;
         this.ignoreAbove = ignoreAbove;
     }
@@ -319,7 +571,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
 
         if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
             Field field = new Field(fieldType().name(), valueAndBoost.value(), fieldType());
-            if (valueAndBoost.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+            if (valueAndBoost.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                 field.setBoost(valueAndBoost.boost());
             }
             fields.add(field);
@@ -346,7 +598,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
             return new ValueAndBoost(nullValue, defaultBoost);
         }
         if (parser.currentToken() == XContentParser.Token.START_OBJECT
-                && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+                && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
             XContentParser.Token token;
             String currentFieldName = null;
             String value = nullValue;
@@ -392,6 +644,11 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
         }
     }
 
+    @Override
+    public StringFieldType fieldType() {
+        return (StringFieldType) super.fieldType();
+    }
+
     @Override
     protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
         super.doXContentBody(builder, includeDefaults, params);
@@ -413,6 +670,27 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
         if (includeDefaults || ignoreAbove != Defaults.IGNORE_ABOVE) {
             builder.field("ignore_above", ignoreAbove);
         }
+        if (includeDefaults || fieldType().fielddata() != ((StringFieldType) defaultFieldType).fielddata()) {
+            builder.field("fielddata", fieldType().fielddata());
+        }
+        if (fieldType().fielddata()) {
+            if (includeDefaults
+                    || fieldType().fielddataMinFrequency() != Defaults.FIELDDATA_MIN_FREQUENCY
+                    || fieldType().fielddataMaxFrequency() != Defaults.FIELDDATA_MAX_FREQUENCY
+                    || fieldType().fielddataMinSegmentSize() != Defaults.FIELDDATA_MIN_SEGMENT_SIZE) {
+                builder.startObject("fielddata_frequency_filter");
+                if (includeDefaults || fieldType().fielddataMinFrequency() != Defaults.FIELDDATA_MIN_FREQUENCY) {
+                    builder.field("min", fieldType().fielddataMinFrequency());
+                }
+                if (includeDefaults || fieldType().fielddataMaxFrequency() != Defaults.FIELDDATA_MAX_FREQUENCY) {
+                    builder.field("max", fieldType().fielddataMaxFrequency());
+                }
+                if (includeDefaults || fieldType().fielddataMinSegmentSize() != Defaults.FIELDDATA_MIN_SEGMENT_SIZE) {
+                    builder.field("min_segment_size", fieldType().fielddataMinSegmentSize());
+                }
+                builder.endObject();
+            }
+        }
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java
index 1ec093ec8e0..0baa4564b69 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java
@@ -27,6 +27,9 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
+import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
@@ -38,6 +41,7 @@ import java.io.IOException;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField;
 import static org.elasticsearch.index.mapper.core.TypeParsers.parseTextField;
@@ -49,6 +53,10 @@ public class TextFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
     private static final int POSITION_INCREMENT_GAP_USE_ANALYZER = -1;
 
     public static class Defaults {
+        public static double FIELDDATA_MIN_FREQUENCY = 0;
+        public static double FIELDDATA_MAX_FREQUENCY = Integer.MAX_VALUE;
+        public static int FIELDDATA_MIN_SEGMENT_SIZE = 0;
+
         public static final MappedFieldType FIELD_TYPE = new TextFieldType();
 
         static {
@@ -72,6 +80,11 @@ public class TextFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
             builder = this;
         }
 
+        @Override
+        public TextFieldType fieldType() {
+            return (TextFieldType) super.fieldType();
+        }
+
         public Builder positionIncrementGap(int positionIncrementGap) {
             if (positionIncrementGap < 0) {
                 throw new MapperParsingException("[positions_increment_gap] must be positive, got " + positionIncrementGap);
@@ -80,6 +93,11 @@ public class TextFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
             return this;
         }
 
+        public Builder fielddata(boolean fielddata) {
+            fieldType().setFielddata(fielddata);
+            return builder;
+        }
+
         @Override
         public Builder docValues(boolean docValues) {
             if (docValues) {
@@ -88,6 +106,18 @@ public class TextFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
             return super.docValues(docValues);
         }
 
+        public Builder eagerGlobalOrdinals(boolean eagerGlobalOrdinals) {
+            fieldType().setEagerGlobalOrdinals(eagerGlobalOrdinals);
+            return builder;
+        }
+
+        public Builder fielddataFrequencyFilter(double minFreq, double maxFreq, int minSegmentSize) {
+            fieldType().setFielddataMinFrequency(minFreq);
+            fieldType().setFielddataMaxFrequency(maxFreq);
+            fieldType().setFielddataMinSegmentSize(minSegmentSize);
+            return builder;
+        }
+
         @Override
         public TextFieldMapper build(BuilderContext context) {
             if (positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) {
@@ -119,7 +149,19 @@ public class TextFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
                     int newPositionIncrementGap = XContentMapValues.nodeIntegerValue(propNode, -1);
                     builder.positionIncrementGap(newPositionIncrementGap);
                     iterator.remove();
-                } else if (parseMultiField(builder, fieldName, parserContext, propName, propNode)) {
+                } else if (propName.equals("fielddata")) {
+                    builder.fielddata(XContentMapValues.nodeBooleanValue(propNode));
+                    iterator.remove();
+                } else if (propName.equals("eager_global_ordinals")) {
+                    builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode));
+                    iterator.remove();
+                } else if (propName.equals("fielddata_frequency_filter")) {
+                    Map<?,?> frequencyFilter = (Map<?, ?>) propNode;
+                    double minFrequency = XContentMapValues.nodeDoubleValue(frequencyFilter.remove("min"), 0);
+                    double maxFrequency = XContentMapValues.nodeDoubleValue(frequencyFilter.remove("max"), Integer.MAX_VALUE);
+                    int minSegmentSize = XContentMapValues.nodeIntegerValue(frequencyFilter.remove("min_segment_size"), 0);
+                    builder.fielddataFrequencyFilter(minFrequency, maxFrequency, minSegmentSize);
+                    DocumentMapperParser.checkNoRemainingFields(propName, frequencyFilter, parserContext.indexVersionCreated());
                     iterator.remove();
                 }
             }
@@ -129,16 +171,110 @@ public class TextFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
 
     public static final class TextFieldType extends MappedFieldType {
 
-        public TextFieldType() {}
+        private boolean fielddata;
+        private double fielddataMinFrequency;
+        private double fielddataMaxFrequency;
+        private int fielddataMinSegmentSize;
+
+        public TextFieldType() {
+            // TODO: change the default to false
+            fielddata = true;
+            fielddataMinFrequency = Defaults.FIELDDATA_MIN_FREQUENCY;
+            fielddataMaxFrequency = Defaults.FIELDDATA_MAX_FREQUENCY;
+            fielddataMinSegmentSize = Defaults.FIELDDATA_MIN_SEGMENT_SIZE;
+        }
 
         protected TextFieldType(TextFieldType ref) {
             super(ref);
+            this.fielddata = ref.fielddata;
+            this.fielddataMinFrequency = ref.fielddataMinFrequency;
+            this.fielddataMaxFrequency = ref.fielddataMaxFrequency;
+            this.fielddataMinSegmentSize = ref.fielddataMinSegmentSize;
         }
 
         public TextFieldType clone() {
             return new TextFieldType(this);
         }
 
+        @Override
+        public boolean equals(Object o) {
+            if (super.equals(o) == false) {
+                return false;
+            }
+            TextFieldType that = (TextFieldType) o;
+            return fielddata == that.fielddata
+                    && fielddataMinFrequency == that.fielddataMinFrequency
+                    && fielddataMaxFrequency == that.fielddataMaxFrequency
+                    && fielddataMinSegmentSize == that.fielddataMinSegmentSize;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(super.hashCode(), fielddata,
+                    fielddataMinFrequency, fielddataMaxFrequency, fielddataMinSegmentSize);
+        }
+
+        @Override
+        public void checkCompatibility(MappedFieldType other,
+                List<String> conflicts, boolean strict) {
+            super.checkCompatibility(other, conflicts, strict);
+            TextFieldType otherType = (TextFieldType) other;
+            if (strict) {
+                if (fielddata() != otherType.fielddata()) {
+                    conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [fielddata] "
+                            + "across all types.");
+                }
+                if (fielddataMinFrequency() != otherType.fielddataMinFrequency()) {
+                    conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update "
+                            + "[fielddata_frequency_filter.min] across all types.");
+                }
+                if (fielddataMaxFrequency() != otherType.fielddataMaxFrequency()) {
+                    conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update "
+                            + "[fielddata_frequency_filter.max] across all types.");
+                }
+                if (fielddataMinSegmentSize() != otherType.fielddataMinSegmentSize()) {
+                    conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update "
+                            + "[fielddata_frequency_filter.min_segment_size] across all types.");
+                }
+            }
+        }
+
+        public boolean fielddata() {
+            return fielddata;
+        }
+
+        public void setFielddata(boolean fielddata) {
+            checkIfFrozen();
+            this.fielddata = fielddata;
+        }
+
+        public double fielddataMinFrequency() {
+            return fielddataMinFrequency;
+        }
+
+        public void setFielddataMinFrequency(double fielddataMinFrequency) {
+            checkIfFrozen();
+            this.fielddataMinFrequency = fielddataMinFrequency;
+        }
+
+        public double fielddataMaxFrequency() {
+            return fielddataMaxFrequency;
+        }
+
+        public void setFielddataMaxFrequency(double fielddataMaxFrequency) {
+            checkIfFrozen();
+            this.fielddataMaxFrequency = fielddataMaxFrequency;
+        }
+
+        public int fielddataMinSegmentSize() {
+            return fielddataMinSegmentSize;
+        }
+
+        public void setFielddataMinSegmentSize(int fielddataMinSegmentSize) {
+            checkIfFrozen();
+            this.fielddataMinSegmentSize = fielddataMinSegmentSize;
+        }
+
         @Override
         public String typeName() {
             return CONTENT_TYPE;
@@ -159,6 +295,16 @@ public class TextFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
             }
             return termQuery(nullValue(), null);
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            if (fielddata == false) {
+                throw new IllegalStateException("Fielddata is disabled on text fields by default. Set fielddata=true on [" + name()
+                        + "] in order to load fielddata in memory by uninverting the inverted index. Note that this can however "
+                        + "use significant memory.");
+            }
+            return new PagedBytesIndexFieldData.Builder(fielddataMinFrequency, fielddataMaxFrequency, fielddataMinSegmentSize);
+        }
     }
 
     private Boolean includeInAll;
@@ -249,6 +395,11 @@ public class TextFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
         this.includeInAll = ((TextFieldMapper) mergeWith).includeInAll;
     }
 
+    @Override
+    public TextFieldType fieldType() {
+        return (TextFieldType) super.fieldType();
+    }
+
     @Override
     protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
         super.doXContentBody(builder, includeDefaults, params);
@@ -263,5 +414,27 @@ public class TextFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
         if (includeDefaults || positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) {
             builder.field("position_increment_gap", positionIncrementGap);
         }
+
+        if (includeDefaults || fieldType().fielddata() != ((TextFieldType) defaultFieldType).fielddata()) {
+            builder.field("fielddata", fieldType().fielddata());
+        }
+        if (fieldType().fielddata()) {
+            if (includeDefaults
+                    || fieldType().fielddataMinFrequency() != Defaults.FIELDDATA_MIN_FREQUENCY
+                    || fieldType().fielddataMaxFrequency() != Defaults.FIELDDATA_MAX_FREQUENCY
+                    || fieldType().fielddataMinSegmentSize() != Defaults.FIELDDATA_MIN_SEGMENT_SIZE) {
+                builder.startObject("fielddata_frequency_filter");
+                if (includeDefaults || fieldType().fielddataMinFrequency() != Defaults.FIELDDATA_MIN_FREQUENCY) {
+                    builder.field("min", fieldType().fielddataMinFrequency());
+                }
+                if (includeDefaults || fieldType().fielddataMaxFrequency() != Defaults.FIELDDATA_MAX_FREQUENCY) {
+                    builder.field("max", fieldType().fielddataMaxFrequency());
+                }
+                if (includeDefaults || fieldType().fielddataMinSegmentSize() != Defaults.FIELDDATA_MIN_SEGMENT_SIZE) {
+                    builder.field("min_segment_size", fieldType().fielddataMinSegmentSize());
+                }
+                builder.endObject();
+            }
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java
index 15fcd9220e2..bc7c97bc4ff 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java
@@ -25,25 +25,27 @@ import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.joda.Joda;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.loader.SettingsLoader;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType.Loading;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
 import org.elasticsearch.index.similarity.SimilarityProvider;
 import org.elasticsearch.index.similarity.SimilarityService;
 
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.isArray;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
@@ -63,10 +65,18 @@ public class TypeParsers {
     public static final String INDEX_OPTIONS_POSITIONS = "positions";
     public static final String INDEX_OPTIONS_OFFSETS = "offsets";
 
-    private static boolean nodeBooleanValue(Object node, Mapper.TypeParser.ParserContext parserContext) {
-        if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0)) {
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(TypeParsers.class));
+    private static final Set<String> BOOLEAN_STRINGS = new HashSet<>(Arrays.asList("true", "false"));
+
+    public static boolean nodeBooleanValue(String name, Object node, Mapper.TypeParser.ParserContext parserContext) {
+        // Hook onto ParseFieldMatcher so that parsing becomes strict when setting index.query.parse.strict
+        if (parserContext.parseFieldMatcher().isStrict()) {
             return XContentMapValues.nodeBooleanValue(node);
         } else {
+            // TODO: remove this leniency in 6.0
+            if (BOOLEAN_STRINGS.contains(node.toString()) == false) {
+                DEPRECATION_LOGGER.deprecated("Expected a boolean for property [{}] but got [{}]", name, node);
+            }
             return XContentMapValues.lenientNodeBooleanValue(node);
         }
     }
@@ -81,13 +91,10 @@ public class TypeParsers {
                 builder.precisionStep(nodeIntegerValue(propNode));
                 iterator.remove();
             } else if (propName.equals("ignore_malformed")) {
-                builder.ignoreMalformed(nodeBooleanValue(propNode, parserContext));
+                builder.ignoreMalformed(nodeBooleanValue("ignore_malformed", propNode, parserContext));
                 iterator.remove();
             } else if (propName.equals("coerce")) {
-                builder.coerce(nodeBooleanValue(propNode, parserContext));
-                iterator.remove();
-            } else if (propName.equals("omit_norms")) {
-                builder.omitNorms(nodeBooleanValue(propNode, parserContext));
+                builder.coerce(nodeBooleanValue("coerce", propNode, parserContext));
                 iterator.remove();
             } else if (propName.equals("similarity")) {
                 SimilarityProvider similarityProvider = resolveSimilarity(parserContext, name, propNode.toString());
@@ -112,16 +119,16 @@ public class TypeParsers {
                 parseTermVector(name, propNode.toString(), builder);
                 iterator.remove();
             } else if (propName.equals("store_term_vectors")) {
-                builder.storeTermVectors(nodeBooleanValue(propNode, parserContext));
+                builder.storeTermVectors(nodeBooleanValue("store_term_vectors", propNode, parserContext));
                 iterator.remove();
             } else if (propName.equals("store_term_vector_offsets")) {
-                builder.storeTermVectorOffsets(nodeBooleanValue(propNode, parserContext));
+                builder.storeTermVectorOffsets(nodeBooleanValue("store_term_vector_offsets", propNode, parserContext));
                 iterator.remove();
             } else if (propName.equals("store_term_vector_positions")) {
-                builder.storeTermVectorPositions(nodeBooleanValue(propNode, parserContext));
+                builder.storeTermVectorPositions(nodeBooleanValue("store_term_vector_positions", propNode, parserContext));
                 iterator.remove();
             } else if (propName.equals("store_term_vector_payloads")) {
-                builder.storeTermVectorPayloads(nodeBooleanValue(propNode, parserContext));
+                builder.storeTermVectorPayloads(nodeBooleanValue("store_term_vector_payloads", propNode, parserContext));
                 iterator.remove();
             } else if (propName.equals("analyzer")) {
                 NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString());
@@ -174,6 +181,37 @@ public class TypeParsers {
         }
     }
 
+    public static boolean parseNorms(FieldMapper.Builder builder, String propName, Object propNode, Mapper.TypeParser.ParserContext parserContext) {
+        if (propName.equals("norms")) {
+            if (propNode instanceof Map) {
+                final Map<String, Object> properties = nodeMapValue(propNode, "norms");
+                for (Iterator<Entry<String, Object>> propsIterator = properties.entrySet().iterator(); propsIterator.hasNext();) {
+                    Entry<String, Object> entry2 = propsIterator.next();
+                    final String propName2 = Strings.toUnderscoreCase(entry2.getKey());
+                    final Object propNode2 = entry2.getValue();
+                    if (propName2.equals("enabled")) {
+                        builder.omitNorms(!lenientNodeBooleanValue(propNode2));
+                        propsIterator.remove();
+                    } else if (propName2.equals("loading")) {
+                        // ignore for bw compat
+                        propsIterator.remove();
+                    }
+                }
+                DocumentMapperParser.checkNoRemainingFields(propName, properties, parserContext.indexVersionCreated());
+                DEPRECATION_LOGGER.deprecated("The [norms{enabled:true/false}] way of specifying norms is deprecated, please use [norms:true/false] instead");
+            } else {
+                builder.omitNorms(nodeBooleanValue("norms", propNode, parserContext) == false);
+            }
+            return true;
+        } else if (propName.equals("omit_norms")) {
+            builder.omitNorms(nodeBooleanValue("norms", propNode, parserContext));
+            DEPRECATION_LOGGER.deprecated("[omit_norms] is deprecated, please use [norms] instead with the opposite boolean value");
+            return true;
+        } else {
+            return false;
+        }
+    }
+
     /**
      * Parse text field attributes. In addition to {@link #parseField common attributes}
      * this will parse analysis and term-vectors related settings.
@@ -181,6 +219,14 @@ public class TypeParsers {
     public static void parseTextField(FieldMapper.Builder builder, String name, Map<String, Object> fieldNode, Mapper.TypeParser.ParserContext parserContext) {
         parseField(builder, name, fieldNode, parserContext);
         parseAnalyzersAndTermVectors(builder, name, fieldNode, parserContext);
+        for (Iterator<Map.Entry<String, Object>> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) {
+            Map.Entry<String, Object> entry = iterator.next();
+            final String propName = Strings.toUnderscoreCase(entry.getKey());
+            final Object propNode = entry.getValue();
+            if (parseNorms(builder, propName, propNode, parserContext)) {
+                iterator.remove();
+            }
+        }
     }
 
     /**
@@ -199,43 +245,30 @@ public class TypeParsers {
                 builder.index(parseIndex(name, propNode.toString(), parserContext));
                 iterator.remove();
             } else if (propName.equals(DOC_VALUES)) {
-                builder.docValues(nodeBooleanValue(propNode, parserContext));
+                builder.docValues(nodeBooleanValue(DOC_VALUES, propNode, parserContext));
                 iterator.remove();
             } else if (propName.equals("boost")) {
                 builder.boost(nodeFloatValue(propNode));
                 iterator.remove();
-            } else if (propName.equals("omit_norms")) {
-                builder.omitNorms(nodeBooleanValue(propNode, parserContext));
-                iterator.remove();
-            } else if (propName.equals("norms")) {
-                final Map<String, Object> properties = nodeMapValue(propNode, "norms");
-                for (Iterator<Entry<String, Object>> propsIterator = properties.entrySet().iterator(); propsIterator.hasNext();) {
-                    Entry<String, Object> entry2 = propsIterator.next();
-                    final String propName2 = Strings.toUnderscoreCase(entry2.getKey());
-                    final Object propNode2 = entry2.getValue();
-                    if (propName2.equals("enabled")) {
-                        builder.omitNorms(!lenientNodeBooleanValue(propNode2));
-                        propsIterator.remove();
-                    } else if (propName2.equals(Loading.KEY)) {
-                        builder.normsLoading(Loading.parse(nodeStringValue(propNode2, null), null));
-                        propsIterator.remove();
-                    }
-                }
-                DocumentMapperParser.checkNoRemainingFields(propName, properties, parserContext.indexVersionCreated());
+            } else if (parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1)
+                    && parseNorms(builder, propName, propNode, parserContext)) {
                 iterator.remove();
             } else if (propName.equals("index_options")) {
                 builder.indexOptions(nodeIndexOptionValue(propNode));
                 iterator.remove();
             } else if (propName.equals("include_in_all")) {
-                builder.includeInAll(nodeBooleanValue(propNode, parserContext));
+                builder.includeInAll(nodeBooleanValue("include_in_all", propNode, parserContext));
                 iterator.remove();
             } else if (propName.equals("similarity")) {
                 SimilarityProvider similarityProvider = resolveSimilarity(parserContext, name, propNode.toString());
                 builder.similarity(similarityProvider);
                 iterator.remove();
-            } else if (propName.equals("fielddata")) {
-                final Settings settings = Settings.builder().put(SettingsLoader.Helper.loadNestedFromMap(nodeMapValue(propNode, "fielddata"))).build();
-                builder.fieldDataSettings(settings);
+            } else if (propName.equals("fielddata")
+                    && propNode instanceof Map
+                    && parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1)) {
+                // ignore for bw compat
+                iterator.remove();
+            } else if (parseMultiField(builder, name, parserContext, propName, propNode)) {
                 iterator.remove();
             } else if (propName.equals("copy_to")) {
                 if (parserContext.isWithinMultiField()) {
@@ -243,7 +276,7 @@ public class TypeParsers {
                         (indexVersionCreated.after(Version.V_2_0_1) && indexVersionCreated.before(Version.V_2_1_0))) {
                         throw new MapperParsingException("copy_to in multi fields is not allowed. Found the copy_to in field [" + name + "] which is within a multi field.");
                     } else {
-                        ESLoggerFactory.getLogger("mapping [" + parserContext.type() + "]").warn("Found a copy_to in field [" + name + "] which is within a multi field. This feature has been removed and the copy_to will be removed from the mapping.");
+                        ESLoggerFactory.getLogger("mapping [" + parserContext.type() + "]").warn("Found a copy_to in field [{}] which is within a multi field. This feature has been removed and the copy_to will be removed from the mapping.", name);
                     }
                 } else {
                     parseCopyFields(propNode, builder);
@@ -353,35 +386,32 @@ public class TypeParsers {
     }
 
     public static boolean parseIndex(String fieldName, String index, Mapper.TypeParser.ParserContext parserContext) throws MapperParsingException {
-        if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0)) {
-            switch (index) {
-            case "true":
-                return true;
-            case "false":
-                return false;
-            default:
+        switch (index) {
+        case "true":
+            return true;
+        case "false":
+            return false;
+        case "not_analyzed":
+        case "analyzed":
+        case "no":
+            if (parserContext.parseFieldMatcher().isStrict() == false) {
+                DEPRECATION_LOGGER.deprecated("Expected a boolean for property [index] but got [{}]", index);
+                return "no".equals(index) == false;
+            } else {
                 throw new IllegalArgumentException("Can't parse [index] value [" + index + "] for field [" + fieldName + "], expected [true] or [false]");
             }
-        } else {
-            final String normalizedIndex = Strings.toUnderscoreCase(index);
-            switch (normalizedIndex) {
-            case "true":
-            case "not_analyzed":
-            case "analyzed":
-                return true;
-            case "false":
-            case "no":
-                return false;
-            default:
-                throw new IllegalArgumentException("Can't parse [index] value [" + index + "] for field [" + fieldName + "], expected [true], [false], [no], [not_analyzed] or [analyzed]");
-            }
+        default:
+            throw new IllegalArgumentException("Can't parse [index] value [" + index + "] for field [" + fieldName + "], expected [true] or [false]");
         }
     }
 
     public static boolean parseStore(String fieldName, String store, Mapper.TypeParser.ParserContext parserContext) throws MapperParsingException {
-        if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0)) {
+        if (parserContext.parseFieldMatcher().isStrict()) {
             return XContentMapValues.nodeBooleanValue(store);
         } else {
+            if (BOOLEAN_STRINGS.contains(store) == false) {
+                DEPRECATION_LOGGER.deprecated("Expected a boolean for property [store] but got [{}]", store);
+            }
             if ("no".equals(store)) {
                 return false;
             } else if ("yes".equals(store)) {
@@ -406,7 +436,7 @@ public class TypeParsers {
     }
 
     private static SimilarityProvider resolveSimilarity(Mapper.TypeParser.ParserContext parserContext, String name, String value) {
-        if (parserContext.indexVersionCreated().before(Version.V_5_0_0) && "default".equals(value)) {
+        if (parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1) && "default".equals(value)) {
             // "default" similarity has been renamed into "classic" in 3.x.
             value = SimilarityService.DEFAULT_SIMILARITY;
         }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java
index 1a1c1592d7e..fec0720ce70 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java
@@ -28,10 +28,14 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.plain.AbstractGeoPointDVIndexFieldData;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
@@ -56,13 +60,13 @@ import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField;
  */
 public abstract class BaseGeoPointFieldMapper extends FieldMapper implements ArrayValueMapperParser {
     public static final String CONTENT_TYPE = "geo_point";
+    protected static final DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(BaseGeoPointFieldMapper.class));
     public static class Names {
         public static final String LAT = "lat";
         public static final String LAT_SUFFIX = "." + LAT;
         public static final String LON = "lon";
         public static final String LON_SUFFIX = "." + LON;
         public static final String GEOHASH = "geohash";
-        public static final String GEOHASH_SUFFIX = "." + GEOHASH;
         public static final String IGNORE_MALFORMED = "ignore_malformed";
     }
 
@@ -71,7 +75,7 @@ public abstract class BaseGeoPointFieldMapper extends FieldMapper implements Arr
         public static final boolean ENABLE_GEOHASH = false;
         public static final boolean ENABLE_GEOHASH_PREFIX = false;
         public static final int GEO_HASH_PRECISION = GeoHashUtils.PRECISION;
-        public static final Explicit<Boolean> IGNORE_MALFORMED = new Explicit(false, false);
+        public static final Explicit<Boolean> IGNORE_MALFORMED = new Explicit<>(false, false);
     }
 
     public abstract static class Builder<T extends Builder, Y extends BaseGeoPointFieldMapper> extends FieldMapper.Builder<T, Y> {
@@ -97,12 +101,6 @@ public abstract class BaseGeoPointFieldMapper extends FieldMapper implements Arr
             return (GeoPointFieldType)fieldType;
         }
 
-        @Override
-        public T fieldDataSettings(Settings settings) {
-            this.fieldDataSettings = settings;
-            return builder;
-        }
-
         public T enableLatLon(boolean enableLatLon) {
             this.enableLatLon = enableLatLon;
             return builder;
@@ -194,9 +192,13 @@ public abstract class BaseGeoPointFieldMapper extends FieldMapper implements Arr
                 String propName = Strings.toUnderscoreCase(entry.getKey());
                 Object propNode = entry.getValue();
                 if (propName.equals("lat_lon")) {
+                    deprecationLogger.deprecated(CONTENT_TYPE + " lat_lon parameter is deprecated and will be removed "
+                        + "in the next major release");
                     builder.enableLatLon(XContentMapValues.lenientNodeBooleanValue(propNode));
                     iterator.remove();
                 } else if (propName.equals("precision_step")) {
+                    deprecationLogger.deprecated(CONTENT_TYPE + " precision_step parameter is deprecated and will be removed "
+                        + "in the next major release");
                     builder.precisionStep(XContentMapValues.nodeIntegerValue(propNode));
                     iterator.remove();
                 } else if (propName.equals("geohash")) {
@@ -218,8 +220,6 @@ public abstract class BaseGeoPointFieldMapper extends FieldMapper implements Arr
                 } else if (propName.equals(Names.IGNORE_MALFORMED)) {
                     builder.ignoreMalformed(XContentMapValues.lenientNodeBooleanValue(propNode));
                     iterator.remove();
-                } else if (parseMultiField(builder, name, parserContext, propName, propNode)) {
-                    iterator.remove();
                 }
             }
 
@@ -339,6 +339,11 @@ public abstract class BaseGeoPointFieldMapper extends FieldMapper implements Arr
             this.latFieldType = latFieldType;
             this.lonFieldType = lonFieldType;
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            return new AbstractGeoPointDVIndexFieldData.Builder();
+        }
     }
 
     protected DoubleFieldMapper latMapper;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java
index 57778fa8d25..1a0f2561a40 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java
@@ -46,7 +46,6 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
-import org.elasticsearch.index.mapper.core.DoubleFieldMapper;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -454,7 +453,7 @@ public class GeoShapeFieldMapper extends FieldMapper {
             }
             for (Field field : fields) {
                 if (!customBoost() &&
-                    fieldType.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+                    fieldType.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                     field.setBoost(fieldType().boost());
                 }
                 context.doc().add(field);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
index 97c2fa3933b..bc523383208 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
@@ -24,7 +24,6 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.lucene.Lucene;
@@ -32,7 +31,6 @@ import org.elasticsearch.common.lucene.all.AllField;
 import org.elasticsearch.common.lucene.all.AllTermQuery;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -168,7 +166,6 @@ public class AllFieldMapper extends MetadataFieldMapper {
     static final class AllFieldType extends MappedFieldType {
 
         public AllFieldType() {
-            setFieldDataType(new FieldDataType("string"));
         }
 
         protected AllFieldType(AllFieldType ref) {
@@ -305,7 +302,7 @@ public class AllFieldMapper extends MetadataFieldMapper {
             builder.field("store_term_vector_payloads", fieldType().storeTermVectorPayloads());
         }
         if (includeDefaults || fieldType().omitNorms() != Defaults.FIELD_TYPE.omitNorms()) {
-            builder.field("omit_norms", fieldType().omitNorms());
+            builder.field("norms", !fieldType().omitNorms());
         }
         
         doXContentAnalyzers(builder, includeDefaults);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java
index 03ebcb9fe95..77dbe358265 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java
@@ -22,16 +22,17 @@ package org.elasticsearch.index.mapper.internal;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.search.Query;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -130,7 +131,6 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper {
         private boolean enabled = Defaults.ENABLED;
 
         public FieldNamesFieldType() {
-            setFieldDataType(new FieldDataType("string"));
         }
 
         protected FieldNamesFieldType(FieldNamesFieldType ref) {
@@ -192,6 +192,14 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper {
         public boolean useTermQueryWithQueryString() {
             return true;
         }
+
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            if (isEnabled() == false) {
+                throw new IllegalStateException("Cannot run [exists] queries if the [_field_names] field is disabled");
+            }
+            return super.termQuery(value, context);
+        }
     }
 
     private FieldNamesFieldMapper(Settings indexSettings, MappedFieldType existing) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
index 1f26dd60841..2a21e8a7c69 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
@@ -38,7 +38,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -112,7 +111,6 @@ public class IdFieldMapper extends MetadataFieldMapper {
     static final class IdFieldType extends MappedFieldType {
 
         public IdFieldType() {
-            setFieldDataType(new FieldDataType("string"));
         }
 
         protected IdFieldType(IdFieldType ref) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
index d4aa2da4ab3..cb3e4ad2193 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
@@ -29,6 +29,8 @@ import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.plain.IndexIndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -184,6 +186,11 @@ public class IndexFieldMapper extends MetadataFieldMapper {
             }
             return value.toString();
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            return new IndexIndexFieldData.Builder();
+        }
     }
 
     private EnabledAttributeMapper enabledState;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
index 66e754e5fda..aedafc80209 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
@@ -30,11 +30,14 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.loader.SettingsLoader;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.fielddata.FieldDataType;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
 import org.elasticsearch.index.mapper.ContentPath;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
@@ -53,7 +56,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeMapValue;
 
 /**
@@ -94,6 +96,11 @@ public class ParentFieldMapper extends MetadataFieldMapper {
             return builder;
         }
 
+        public Builder eagerGlobalOrdinals(boolean eagerGlobalOrdinals) {
+            ((ParentFieldType) fieldType()).setEagerGlobalOrdinals(eagerGlobalOrdinals);
+            return builder;
+        }
+
         @Override
         public ParentFieldMapper build(BuilderContext context) {
             if (parentType == null) {
@@ -106,6 +113,7 @@ public class ParentFieldMapper extends MetadataFieldMapper {
     }
 
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
+        private static final ParseField FIELDDATA = new ParseField("fielddata").withAllDeprecated("eager_global_ordinals");
         @Override
         public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             Builder builder = new Builder(parserContext.type());
@@ -116,14 +124,16 @@ public class ParentFieldMapper extends MetadataFieldMapper {
                 if (fieldName.equals("type")) {
                     builder.type(fieldNode.toString());
                     iterator.remove();
-                } else if (fieldName.equals("fielddata")) {
-                    // Only take over `loading`, since that is the only option now that is configurable:
+                } else if (parserContext.parseFieldMatcher().match(fieldName, FIELDDATA)) {
+                    // for bw compat only
                     Map<String, String> fieldDataSettings = SettingsLoader.Helper.loadNestedFromMap(nodeMapValue(fieldNode, "fielddata"));
-                    if (fieldDataSettings.containsKey(MappedFieldType.Loading.KEY)) {
-                        Settings settings = settingsBuilder().put(MappedFieldType.Loading.KEY, fieldDataSettings.get(MappedFieldType.Loading.KEY)).build();
-                        builder.fieldDataSettings(settings);
+                    if (fieldDataSettings.containsKey("loading")) {
+                        builder.eagerGlobalOrdinals("eager_global_ordinals".equals(fieldDataSettings.get("loading")));
                     }
                     iterator.remove();
+                } else if (fieldName.equals("eager_global_ordinals")) {
+                    builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(fieldNode));
+                    iterator.remove();
                 }
             }
             return builder;
@@ -143,7 +153,6 @@ public class ParentFieldMapper extends MetadataFieldMapper {
         parentJoinField.indexOptions(IndexOptions.NONE);
         parentJoinField.docValues(true);
         parentJoinField.fieldType().setDocValuesType(DocValuesType.SORTED);
-        parentJoinField.fieldType().setFieldDataType(null);
         return parentJoinField.build(context);
     }
 
@@ -152,8 +161,8 @@ public class ParentFieldMapper extends MetadataFieldMapper {
         final String documentType;
 
         public ParentFieldType() {
-            setFieldDataType(new FieldDataType(NAME, settingsBuilder().put(MappedFieldType.Loading.KEY, Loading.EAGER_VALUE)));
             documentType = null;
+            setEagerGlobalOrdinals(true);
         }
 
         ParentFieldType(ParentFieldType ref, String documentType) {
@@ -200,6 +209,11 @@ public class ParentFieldMapper extends MetadataFieldMapper {
             query.add(new TermQuery(new Term(TypeFieldMapper.NAME, documentType)), BooleanClause.Occur.FILTER);
             return query.build();
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            return new ParentChildIndexFieldData.Builder();
+        }
     }
 
     private final String parentType;
@@ -288,17 +302,13 @@ public class ParentFieldMapper extends MetadataFieldMapper {
 
         builder.startObject(CONTENT_TYPE);
         builder.field("type", parentType);
-        if (includeDefaults || joinFieldHasCustomFieldDataSettings()) {
-            builder.field("fielddata", (Map) fieldType().fieldDataType().getSettings().getAsMap());
+        if (includeDefaults || fieldType().eagerGlobalOrdinals() != defaultFieldType.eagerGlobalOrdinals()) {
+            builder.field("eager_global_ordinals", fieldType().eagerGlobalOrdinals());
         }
         builder.endObject();
         return builder;
     }
 
-    private boolean joinFieldHasCustomFieldDataSettings() {
-        return fieldType != null && fieldType.fieldDataType() != null && fieldType.fieldDataType().equals(Defaults.FIELD_TYPE.fieldDataType()) == false;
-    }
-
     @Override
     protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
         super.doMerge(mergeWith, updateAllTypes);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
index b1d24e53ab5..4954445da66 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
@@ -26,7 +26,6 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -111,7 +110,6 @@ public class RoutingFieldMapper extends MetadataFieldMapper {
     static final class RoutingFieldType extends MappedFieldType {
 
         public RoutingFieldType() {
-            setFieldDataType(new FieldDataType("string"));
         }
 
         protected RoutingFieldType(RoutingFieldType ref) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
index a102f03a5f1..3eeca0352a7 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
@@ -124,7 +124,7 @@ public class SourceFieldMapper extends MetadataFieldMapper {
                 if (fieldName.equals("enabled")) {
                     builder.enabled(lenientNodeBooleanValue(fieldNode));
                     iterator.remove();
-                } else if ("format".equals(fieldName) && parserContext.indexVersionCreated().before(Version.V_5_0_0)) {
+                } else if ("format".equals(fieldName) && parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1)) {
                     // ignore on old indices, reject on and after 5.0
                     iterator.remove();
                 } else if (fieldName.equals("includes")) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
index 7c51b05cb4b..fc1e2a6a314 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper.internal;
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -95,7 +94,7 @@ public class TTLFieldMapper extends MetadataFieldMapper {
         public TTLFieldMapper build(BuilderContext context) {
             setupFieldType(context);
             fieldType.setHasDocValues(false);
-            return new TTLFieldMapper(fieldType, enabledState, defaultTTL, fieldDataSettings, context.indexSettings());
+            return new TTLFieldMapper(fieldType, enabledState, defaultTTL, context.indexSettings());
         }
     }
 
@@ -161,11 +160,11 @@ public class TTLFieldMapper extends MetadataFieldMapper {
     private long defaultTTL;
 
     private TTLFieldMapper(Settings indexSettings) {
-        this(Defaults.TTL_FIELD_TYPE.clone(), Defaults.ENABLED_STATE, Defaults.DEFAULT, null, indexSettings);
+        this(Defaults.TTL_FIELD_TYPE.clone(), Defaults.ENABLED_STATE, Defaults.DEFAULT, indexSettings);
     }
 
     private TTLFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabled, long defaultTTL,
-                             @Nullable Settings fieldDataSettings, Settings indexSettings) {
+                             Settings indexSettings) {
         super(NAME, fieldType, Defaults.TTL_FIELD_TYPE, indexSettings);
         this.enabledState = enabled;
         this.defaultTTL = defaultTTL;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
index 72defadf6fd..b9e3434fc25 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
@@ -34,7 +34,6 @@ import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -102,7 +101,6 @@ public class TypeFieldMapper extends MetadataFieldMapper {
     static final class TypeFieldType extends MappedFieldType {
 
         public TypeFieldType() {
-            setFieldDataType(new FieldDataType("string"));
         }
 
         protected TypeFieldType(TypeFieldType ref) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
index f8fea4071e5..84aaedefbec 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
@@ -25,25 +25,24 @@ import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.fielddata.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParseContext.Document;
+import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.index.mapper.Uid;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.index.mapper.core.TypeParsers.parseField;
-
 /**
  *
  */
@@ -104,7 +103,6 @@ public class UidFieldMapper extends MetadataFieldMapper {
     static final class UidFieldType extends MappedFieldType {
 
         public UidFieldType() {
-            setFieldDataType(new FieldDataType("string"));
         }
 
         protected UidFieldType(UidFieldType ref) {
@@ -128,6 +126,15 @@ public class UidFieldMapper extends MetadataFieldMapper {
             }
             return Uid.createUid(value.toString());
         }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            // TODO: add doc values support?
+            return new PagedBytesIndexFieldData.Builder(
+                    TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY,
+                    TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY,
+                    TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE);
+        }
     }
 
     private UidFieldMapper(Settings indexSettings, MappedFieldType existing) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java
index 027b2ef05ff..d019c4768b1 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java
@@ -24,7 +24,6 @@ import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.DocValuesType;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -82,7 +81,6 @@ public class VersionFieldMapper extends MetadataFieldMapper {
     static final class VersionFieldType extends MappedFieldType {
 
         public VersionFieldType() {
-            setFieldDataType(new FieldDataType("long"));
         }
 
         protected VersionFieldType(VersionFieldType ref) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
index 9a4cf70782b..b754c163669 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
@@ -22,12 +22,14 @@ package org.elasticsearch.index.mapper.ip;
 import org.apache.lucene.analysis.LegacyNumericTokenStream;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.LegacyNumericRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.LegacyNumericUtils;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Numbers;
@@ -41,7 +43,9 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericAnalyzer;
 import org.elasticsearch.index.analysis.NumericTokenizer;
-import org.elasticsearch.index.fielddata.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -160,7 +164,6 @@ public class IpFieldMapper extends NumberFieldMapper {
     public static final class IpFieldType extends LongFieldMapper.LongFieldType {
 
         public IpFieldType() {
-            setFieldDataType(new FieldDataType("long"));
         }
 
         protected IpFieldType(IpFieldType ref) {
@@ -262,6 +265,19 @@ public class IpFieldMapper extends NumberFieldMapper {
                 iValue + iSim,
                 true, true);
         }
+
+        @Override
+        public FieldStats stats(Terms terms, int maxDoc) throws IOException {
+            long minValue = LegacyNumericUtils.getMinLong(terms);
+            long maxValue = LegacyNumericUtils.getMaxLong(terms);
+            return new FieldStats.Ip(maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue);
+        }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            failIfNoDocValues();
+            return new DocValuesIndexFieldData.Builder().numericType(NumericType.LONG);
+        }
     }
 
     protected IpFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
@@ -306,7 +322,7 @@ public class IpFieldMapper extends NumberFieldMapper {
         final long value = ipToLong(ipAsString);
         if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
             CustomLongNumericField field = new CustomLongNumericField(value, fieldType());
-            if (fieldType.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
+            if (fieldType.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) {
                 field.setBoost(fieldType().boost());
             }
             fields.add(field);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java b/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java
index 58602f06dfa..3deb487e50b 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java
@@ -19,11 +19,15 @@
 
 package org.elasticsearch.index.mapper.object;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.mapper.ContentPath;
 import org.elasticsearch.index.mapper.MapperParsingException;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -33,30 +37,52 @@ import java.util.TreeMap;
 /**
  *
  */
-public class DynamicTemplate {
+public class DynamicTemplate implements ToXContent {
 
     public static enum MatchType {
-        SIMPLE,
-        REGEX;
+        SIMPLE {
+            @Override
+            public boolean matches(String pattern, String value) {
+                return Regex.simpleMatch(pattern, value);
+            }
+            @Override
+            public String toString() {
+                return "simple";
+            }
+        },
+        REGEX {
+            @Override
+            public boolean matches(String pattern, String value) {
+                return value.matches(pattern);
+            }
+            @Override
+            public String toString() {
+                return "regex";
+            }
+        };
 
         public static MatchType fromString(String value) {
-            if ("simple".equals(value)) {
-                return SIMPLE;
-            } else if ("regex".equals(value)) {
-                return REGEX;
+            for (MatchType v : values()) {
+                if (v.toString().equals(value)) {
+                    return v;
+                }
             }
             throw new IllegalArgumentException("No matching pattern matched on [" + value + "]");
         }
+
+        /** Whether {@code value} matches {@code regex}. */
+        public abstract boolean matches(String regex, String value);
     }
 
-    public static DynamicTemplate parse(String name, Map<String, Object> conf) throws MapperParsingException {
+    public static DynamicTemplate parse(String name, Map<String, Object> conf,
+            Version indexVersionCreated) throws MapperParsingException {
         String match = null;
         String pathMatch = null;
         String unmatch = null;
         String pathUnmatch = null;
         Map<String, Object> mapping = null;
         String matchMappingType = null;
-        String matchPattern = "simple";
+        String matchPattern = MatchType.SIMPLE.toString();
 
         for (Map.Entry<String, Object> entry : conf.entrySet()) {
             String propName = Strings.toUnderscoreCase(entry.getKey());
@@ -74,22 +100,18 @@ public class DynamicTemplate {
                 matchPattern = entry.getValue().toString();
             } else if ("mapping".equals(propName)) {
                 mapping = (Map<String, Object>) entry.getValue();
+            } else if (indexVersionCreated.onOrAfter(Version.V_5_0_0_alpha1)) {
+                // unknown parameters were ignored before but still carried through serialization
+                // so we need to ignore them at parsing time for old indices
+                throw new IllegalArgumentException("Illegal dynamic template parameter: [" + propName + "]");
             }
         }
 
-        if (match == null && pathMatch == null && matchMappingType == null) {
-            throw new MapperParsingException("template must have match, path_match or match_mapping_type set");
-        }
-        if (mapping == null) {
-            throw new MapperParsingException("template must have mapping set");
-        }
-        return new DynamicTemplate(name, conf, pathMatch, pathUnmatch, match, unmatch, matchMappingType, MatchType.fromString(matchPattern), mapping);
+        return new DynamicTemplate(name, pathMatch, pathUnmatch, match, unmatch, matchMappingType, MatchType.fromString(matchPattern), mapping);
     }
 
     private final String name;
 
-    private final Map<String, Object> conf;
-
     private final String pathMatch;
 
     private final String pathUnmatch;
@@ -104,9 +126,14 @@ public class DynamicTemplate {
 
     private final Map<String, Object> mapping;
 
-    public DynamicTemplate(String name, Map<String, Object> conf, String pathMatch, String pathUnmatch, String match, String unmatch, String matchMappingType, MatchType matchType, Map<String, Object> mapping) {
+    public DynamicTemplate(String name, String pathMatch, String pathUnmatch, String match, String unmatch, String matchMappingType, MatchType matchType, Map<String, Object> mapping) {
+        if (match == null && pathMatch == null && matchMappingType == null) {
+            throw new MapperParsingException("template must have match, path_match or match_mapping_type set");
+        }
+        if (mapping == null) {
+            throw new MapperParsingException("template must have mapping set");
+        }
         this.name = name;
-        this.conf = new TreeMap<>(conf);
         this.pathMatch = pathMatch;
         this.pathUnmatch = pathUnmatch;
         this.match = match;
@@ -120,48 +147,55 @@ public class DynamicTemplate {
         return this.name;
     }
 
-    public Map<String, Object> conf() {
-        return this.conf;
-    }
-
     public boolean match(ContentPath path, String name, String dynamicType) {
-        if (pathMatch != null && !patternMatch(pathMatch, path.pathAsText(name))) {
+        if (pathMatch != null && !matchType.matches(pathMatch, path.pathAsText(name))) {
             return false;
         }
-        if (match != null && !patternMatch(match, name)) {
+        if (match != null && !matchType.matches(match, name)) {
             return false;
         }
-        if (pathUnmatch != null && patternMatch(pathUnmatch, path.pathAsText(name))) {
+        if (pathUnmatch != null && matchType.matches(pathUnmatch, path.pathAsText(name))) {
             return false;
         }
-        if (unmatch != null && patternMatch(unmatch, name)) {
+        if (unmatch != null && matchType.matches(unmatch, name)) {
             return false;
         }
         if (matchMappingType != null) {
             if (dynamicType == null) {
                 return false;
             }
-            if (!patternMatch(matchMappingType, dynamicType)) {
+            if (!matchType.matches(matchMappingType, dynamicType)) {
                 return false;
             }
         }
         return true;
     }
 
-    public boolean hasType() {
-        return mapping.containsKey("type");
-    }
-
     public String mappingType(String dynamicType) {
-        return mapping.containsKey("type") ? mapping.get("type").toString().replace("{dynamic_type}", dynamicType).replace("{dynamicType}", dynamicType) : dynamicType;
-    }
-
-    private boolean patternMatch(String pattern, String str) {
-        if (matchType == MatchType.SIMPLE) {
-            return Regex.simpleMatch(pattern, str);
+        String type;
+        if (mapping.containsKey("type")) {
+            type = mapping.get("type").toString();
+            type = type.replace("{dynamic_type}", dynamicType);
+            type = type.replace("{dynamicType}", dynamicType);
+        } else {
+            type = dynamicType;
         }
-        return str.matches(pattern);
-    }
+        if (type.equals(mapping.get("type")) == false // either the type was not set, or we updated it through replacements
+                && "text".equals(type)) { // and the result is "text"
+            // now that string has been splitted into text and keyword, we use text for
+            // dynamic mappings. However before it used to be possible to index as a keyword
+            // by setting index=not_analyzed, so for now we will use a keyword field rather
+            // than a text field if index=not_analyzed and the field type was not specified
+            // explicitly
+            // TODO: remove this in 6.0
+            // TODO: how to do it in the future?
+            final Object index = mapping.get("index");
+            if ("not_analyzed".equals(index) || "no".equals(index)) {
+                type = "keyword";
+            }
+        }
+        return type;
+     }
 
     public Map<String, Object> mappingForName(String name, String dynamicType) {
         return processMap(mapping, name, dynamicType);
@@ -200,40 +234,29 @@ public class DynamicTemplate {
     }
 
     @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (match != null) {
+            builder.field("match", match);
         }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
+        if (pathMatch != null) {
+            builder.field("path_match", pathMatch);
         }
-
-        DynamicTemplate that = (DynamicTemplate) o;
-
-        // check if same matching, if so, replace the mapping
-        if (match != null ? !match.equals(that.match) : that.match != null) {
-            return false;
+        if (unmatch != null) {
+            builder.field("unmatch", unmatch);
         }
-        if (matchMappingType != null ? !matchMappingType.equals(that.matchMappingType) : that.matchMappingType != null) {
-            return false;
+        if (pathUnmatch != null) {
+            builder.field("path_unmatch", pathUnmatch);
         }
-        if (matchType != that.matchType) {
-            return false;
+        if (matchMappingType != null) {
+            builder.field("match_mapping_type", matchMappingType);
         }
-        if (unmatch != null ? !unmatch.equals(that.unmatch) : that.unmatch != null) {
-            return false;
+        if (matchType != MatchType.SIMPLE) {
+            builder.field("match_pattern", matchType);
         }
-
-        return true;
-    }
-
-    @Override
-    public int hashCode() {
-        // check if same matching, if so, replace the mapping
-        int result = match != null ? match.hashCode() : 0;
-        result = 31 * result + (unmatch != null ? unmatch.hashCode() : 0);
-        result = 31 * result + (matchType != null ? matchType.hashCode() : 0);
-        result = 31 * result + (matchMappingType != null ? matchMappingType.hashCode() : 0);
-        return result;
+        // use a sorted map for consistent serialization
+        builder.field("mapping", new TreeMap<>(mapping));
+        builder.endObject();
+        return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java
index 00de61acdb6..7e5dc3d28f5 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.mapper.object;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
@@ -140,14 +141,15 @@ public class RootObjectMapper extends ObjectMapper {
                 String fieldName = Strings.toUnderscoreCase(entry.getKey());
                 Object fieldNode = entry.getValue();
                 if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder)
-                        || processField(builder, fieldName, fieldNode)) {
+                        || processField(builder, fieldName, fieldNode, parserContext.indexVersionCreated())) {
                     iterator.remove();
                 }
             }
             return builder;
         }
 
-        protected boolean processField(ObjectMapper.Builder builder, String fieldName, Object fieldNode) {
+        protected boolean processField(ObjectMapper.Builder builder, String fieldName, Object fieldNode,
+                Version indexVersionCreated) {
             if (fieldName.equals("date_formats") || fieldName.equals("dynamic_date_formats")) {
                 List<FormatDateTimeFormatter> dateTimeFormatters = new ArrayList<>();
                 if (fieldNode instanceof List) {
@@ -185,7 +187,10 @@ public class RootObjectMapper extends ObjectMapper {
                         throw new MapperParsingException("A dynamic template must be defined with a name");
                     }
                     Map.Entry<String, Object> entry = tmpl.entrySet().iterator().next();
-                    ((Builder) builder).add(DynamicTemplate.parse(entry.getKey(), (Map<String, Object>) entry.getValue()));
+                    String templateName = entry.getKey();
+                    Map<String, Object> templateParams = (Map<String, Object>) entry.getValue();
+                    DynamicTemplate template = DynamicTemplate.parse(templateName, templateParams, indexVersionCreated);
+                    ((Builder) builder).add(template);
                 }
                 return true;
             } else if (fieldName.equals("date_detection")) {
@@ -329,8 +334,7 @@ public class RootObjectMapper extends ObjectMapper {
             builder.startArray("dynamic_templates");
             for (DynamicTemplate dynamicTemplate : dynamicTemplates) {
                 builder.startObject();
-                builder.field(dynamicTemplate.name());
-                builder.map(dynamicTemplate.conf());
+                builder.field(dynamicTemplate.name(), dynamicTemplate);
                 builder.endObject();
             }
             builder.endArray();
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java b/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java
index 8ee9d55c312..3218837261b 100644
--- a/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java
+++ b/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java
@@ -23,9 +23,12 @@ import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.PrefixCodedTerms;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.queries.BlendedTermQuery;
+import org.apache.lucene.queries.CommonTermsQuery;
 import org.apache.lucene.queries.TermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
@@ -92,10 +95,17 @@ public final class ExtractQueryTermsService {
      * an UnsupportedQueryException is thrown.
      */
     static Set<Term> extractQueryTerms(Query query) {
-        // TODO: add support for the TermsQuery when it has methods to access the actual terms it encapsulates
         // TODO: add support for span queries
         if (query instanceof TermQuery) {
             return Collections.singleton(((TermQuery) query).getTerm());
+        } else if (query instanceof TermsQuery) {
+            Set<Term> terms = new HashSet<>();
+            TermsQuery termsQuery = (TermsQuery) query;
+            PrefixCodedTerms.TermIterator iterator = termsQuery.getTermData().iterator();
+            for (BytesRef term = iterator.next(); term != null; term = iterator.next()) {
+                terms.add(new Term(iterator.field(), term));
+            }
+            return  terms;
         } else if (query instanceof PhraseQuery) {
             Term[] terms = ((PhraseQuery) query).getTerms();
             if (terms.length == 0) {
@@ -154,6 +164,12 @@ public final class ExtractQueryTermsService {
         } else if (query instanceof BoostQuery) {
             Query wrappedQuery = ((BoostQuery) query).getQuery();
             return extractQueryTerms(wrappedQuery);
+        } else if (query instanceof CommonTermsQuery) {
+            List<Term> terms = ((CommonTermsQuery) query).getTerms();
+            return new HashSet<>(terms);
+        } else if (query instanceof BlendedTermQuery) {
+            List<Term> terms = ((BlendedTermQuery) query).getTerms();
+            return new HashSet<>(terms);
         } else {
             throw new UnsupportedQueryException(query);
         }
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolateStats.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolateStats.java
deleted file mode 100644
index f4c899dff9a..00000000000
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolateStats.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.percolator;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentBuilderString;
-
-import java.io.IOException;
-
-/**
- * Exposes percolator related statistics.
- */
-public class PercolateStats implements Streamable, ToXContent {
-
-    private long percolateCount;
-    private long percolateTimeInMillis;
-    private long current;
-    private long memorySizeInBytes = -1;
-    private long numQueries;
-
-    /**
-     * Noop constructor for serialization purposes.
-     */
-    public PercolateStats() {
-    }
-
-    PercolateStats(long percolateCount, long percolateTimeInMillis, long current, long memorySizeInBytes, long numQueries) {
-        this.percolateCount = percolateCount;
-        this.percolateTimeInMillis = percolateTimeInMillis;
-        this.current = current;
-        this.memorySizeInBytes = memorySizeInBytes;
-        this.numQueries = numQueries;
-    }
-
-    /**
-     * @return The number of times the percolate api has been invoked.
-     */
-    public long getCount() {
-        return percolateCount;
-    }
-
-    /**
-     * @return The total amount of time spend in the percolate api
-     */
-    public long getTimeInMillis() {
-        return percolateTimeInMillis;
-    }
-
-    /**
-     * @return The total amount of time spend in the percolate api
-     */
-    public TimeValue getTime() {
-        return new TimeValue(getTimeInMillis());
-    }
-
-    /**
-     * @return The total amount of active percolate api invocations.
-     */
-    public long getCurrent() {
-        return current;
-    }
-
-    /**
-     * @return The total number of loaded percolate queries.
-     */
-    public long getNumQueries() {
-        return numQueries;
-    }
-
-    /**
-     * @return Temporarily returns <code>-1</code>, but this used to return the total size the loaded queries take in
-     * memory, but this is disabled now because the size estimation was too expensive cpu wise. This will be enabled
-     * again when a cheaper size estimation can be found.
-     */
-    public long getMemorySizeInBytes() {
-        return memorySizeInBytes;
-    }
-
-    /**
-     * @return The total size the loaded queries take in memory.
-     */
-    public ByteSizeValue getMemorySize() {
-        return new ByteSizeValue(memorySizeInBytes);
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(Fields.PERCOLATE);
-        builder.field(Fields.TOTAL, percolateCount);
-        builder.timeValueField(Fields.TIME_IN_MILLIS, Fields.TIME, percolateTimeInMillis);
-        builder.field(Fields.CURRENT, current);
-        builder.field(Fields.MEMORY_SIZE_IN_BYTES, memorySizeInBytes);
-        builder.field(Fields.MEMORY_SIZE, getMemorySize());
-        builder.field(Fields.QUERIES, getNumQueries());
-        builder.endObject();
-        return builder;
-    }
-
-    public void add(PercolateStats percolate) {
-        if (percolate == null) {
-            return;
-        }
-
-        percolateCount += percolate.getCount();
-        percolateTimeInMillis += percolate.getTimeInMillis();
-        current += percolate.getCurrent();
-        numQueries += percolate.getNumQueries();
-    }
-
-    static final class Fields {
-        static final XContentBuilderString PERCOLATE = new XContentBuilderString("percolate");
-        static final XContentBuilderString TOTAL = new XContentBuilderString("total");
-        static final XContentBuilderString TIME = new XContentBuilderString("time");
-        static final XContentBuilderString TIME_IN_MILLIS = new XContentBuilderString("time_in_millis");
-        static final XContentBuilderString CURRENT = new XContentBuilderString("current");
-        static final XContentBuilderString MEMORY_SIZE_IN_BYTES = new XContentBuilderString("memory_size_in_bytes");
-        static final XContentBuilderString MEMORY_SIZE = new XContentBuilderString("memory_size");
-        static final XContentBuilderString QUERIES = new XContentBuilderString("queries");
-    }
-
-    public static PercolateStats readPercolateStats(StreamInput in) throws IOException {
-        PercolateStats stats = new PercolateStats();
-        stats.readFrom(in);
-        return stats;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        percolateCount = in.readVLong();
-        percolateTimeInMillis = in.readVLong();
-        current = in.readVLong();
-        numQueries = in.readVLong();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeVLong(percolateCount);
-        out.writeVLong(percolateTimeInMillis);
-        out.writeVLong(current);
-        out.writeVLong(numQueries);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
index f44d454655e..338de5c333d 100644
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
@@ -19,34 +19,50 @@
 package org.elasticsearch.index.percolator;
 
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.core.BinaryFieldMapper;
 import org.elasticsearch.index.mapper.core.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.core.StringFieldMapper;
+import org.elasticsearch.index.query.ParsedQuery;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
 public class PercolatorFieldMapper extends FieldMapper {
 
+    public static final String TYPE_NAME = ".percolator";
     public static final String NAME = "query";
     public static final String CONTENT_TYPE = "percolator";
     public static final PercolatorFieldType FIELD_TYPE = new PercolatorFieldType();
 
     private static final String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms";
     private static final String UNKNOWN_QUERY_FIELD_NAME = "unknown_query";
+    private static final String QUERY_BUILDER_FIELD_NAME = "query_builder_field";
+
     public static final String EXTRACTED_TERMS_FULL_FIELD_NAME = NAME + "." + EXTRACTED_TERMS_FIELD_NAME;
     public static final String UNKNOWN_QUERY_FULL_FIELD_NAME = NAME + "." + UNKNOWN_QUERY_FIELD_NAME;
+    public static final String QUERY_BUILDER_FULL_FIELD_NAME = NAME + "." + QUERY_BUILDER_FIELD_NAME;
 
     public static class Builder extends FieldMapper.Builder<Builder, PercolatorFieldMapper> {
 
@@ -60,19 +76,29 @@ public class PercolatorFieldMapper extends FieldMapper {
         @Override
         public PercolatorFieldMapper build(BuilderContext context) {
             context.path().add(name);
-            KeywordFieldMapper extractedTermsField = createStringFieldBuilder(EXTRACTED_TERMS_FIELD_NAME).build(context);
-            KeywordFieldMapper unknownQueryField = createStringFieldBuilder(UNKNOWN_QUERY_FIELD_NAME).build(context);
+            KeywordFieldMapper extractedTermsField = createExtractQueryFieldBuilder(EXTRACTED_TERMS_FIELD_NAME).build(context);
+            KeywordFieldMapper unknownQueryField = createExtractQueryFieldBuilder(UNKNOWN_QUERY_FIELD_NAME).build(context);
+            BinaryFieldMapper queryBuilderField = createQueryBuilderFieldBuilder().build(context);
             context.path().remove();
-            return new PercolatorFieldMapper(name(), fieldType, defaultFieldType, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo, queryShardContext, extractedTermsField, unknownQueryField);
+            return new PercolatorFieldMapper(name(), fieldType, defaultFieldType, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo, queryShardContext, extractedTermsField, unknownQueryField, queryBuilderField);
         }
 
-        static KeywordFieldMapper.Builder createStringFieldBuilder(String name) {
+        static KeywordFieldMapper.Builder createExtractQueryFieldBuilder(String name) {
             KeywordFieldMapper.Builder queryMetaDataFieldBuilder = new KeywordFieldMapper.Builder(name);
             queryMetaDataFieldBuilder.docValues(false);
             queryMetaDataFieldBuilder.store(false);
             queryMetaDataFieldBuilder.indexOptions(IndexOptions.DOCS);
             return queryMetaDataFieldBuilder;
         }
+
+        static BinaryFieldMapper.Builder createQueryBuilderFieldBuilder() {
+            BinaryFieldMapper.Builder builder = new BinaryFieldMapper.Builder(QUERY_BUILDER_FIELD_NAME);
+            builder.docValues(true);
+            builder.indexOptions(IndexOptions.NONE);
+            builder.store(false);
+            builder.fieldType().setDocValuesType(DocValuesType.BINARY);
+            return builder;
+        }
     }
 
     public static class TypeParser implements FieldMapper.TypeParser {
@@ -111,26 +137,81 @@ public class PercolatorFieldMapper extends FieldMapper {
     private final QueryShardContext queryShardContext;
     private final KeywordFieldMapper queryTermsField;
     private final KeywordFieldMapper unknownQueryField;
+    private final BinaryFieldMapper queryBuilderField;
 
-    public PercolatorFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Settings indexSettings, MultiFields multiFields, CopyTo copyTo, QueryShardContext queryShardContext, KeywordFieldMapper queryTermsField, KeywordFieldMapper unknownQueryField) {
+    public PercolatorFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
+                                 Settings indexSettings, MultiFields multiFields, CopyTo copyTo, QueryShardContext queryShardContext,
+                                 KeywordFieldMapper queryTermsField, KeywordFieldMapper unknownQueryField,
+                                 BinaryFieldMapper queryBuilderField) {
         super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
         this.queryShardContext = queryShardContext;
         this.queryTermsField = queryTermsField;
         this.unknownQueryField = unknownQueryField;
-        this.mapUnmappedFieldAsString = PercolatorQueriesRegistry.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING.get(indexSettings);
+        this.queryBuilderField = queryBuilderField;
+        this.mapUnmappedFieldAsString = PercolatorQueryCache.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING.get(indexSettings);
     }
 
     @Override
     public Mapper parse(ParseContext context) throws IOException {
         QueryShardContext queryShardContext = new QueryShardContext(this.queryShardContext);
-        Query query = PercolatorQueriesRegistry.parseQuery(queryShardContext, mapUnmappedFieldAsString, context.parser());
+        QueryBuilder<?> queryBuilder = parseQueryBuilder(queryShardContext.parseContext(), context.parser());
+        // Fetching of terms, shapes and indexed scripts happen during this rewrite:
+        queryBuilder = queryBuilder.rewrite(queryShardContext);
+
+        try (XContentBuilder builder = XContentFactory.contentBuilder(PercolatorQueryCache.QUERY_BUILDER_CONTENT_TYPE)) {
+            queryBuilder.toXContent(builder, new MapParams(Collections.emptyMap()));
+            builder.flush();
+            byte[] queryBuilderAsBytes = builder.bytes().toBytes();
+            context.doc().add(new Field(queryBuilderField.name(), queryBuilderAsBytes, queryBuilderField.fieldType()));
+        }
+
+        Query query = toQuery(queryShardContext, mapUnmappedFieldAsString, queryBuilder);
         ExtractQueryTermsService.extractQueryTerms(query, context.doc(), queryTermsField.name(), unknownQueryField.name(), queryTermsField.fieldType());
         return null;
     }
 
+    public static Query parseQuery(QueryShardContext context, boolean mapUnmappedFieldsAsString, XContentParser parser) throws IOException {
+        return toQuery(context, mapUnmappedFieldsAsString, parseQueryBuilder(context.parseContext(), parser));
+    }
+
+    static Query toQuery(QueryShardContext context, boolean mapUnmappedFieldsAsString, QueryBuilder<?> queryBuilder) throws IOException {
+        context.reset();
+        // This means that fields in the query need to exist in the mapping prior to registering this query
+        // The reason that this is required, is that if a field doesn't exist then the query assumes defaults, which may be undesired.
+        //
+        // Even worse when fields mentioned in percolator queries do go added to map after the queries have been registered
+        // then the percolator queries don't work as expected any more.
+        //
+        // Query parsing can't introduce new fields in mappings (which happens when registering a percolator query),
+        // because field type can't be inferred from queries (like document do) so the best option here is to disallow
+        // the usage of unmapped fields in percolator queries to avoid unexpected behaviour
+        //
+        // if index.percolator.map_unmapped_fields_as_string is set to true, query can contain unmapped fields which will be mapped
+        // as an analyzed string.
+        context.setAllowUnmappedFields(false);
+        context.setMapUnmappedFieldAsString(mapUnmappedFieldsAsString);
+        context.parseFieldMatcher(context.getIndexSettings().getParseFieldMatcher());
+        try {
+            return queryBuilder.toQuery(context);
+        } finally {
+            context.reset();
+        }
+    }
+
+    static QueryBuilder<?> parseQueryBuilder(QueryParseContext context, XContentParser parser) {
+        context.reset(parser);
+        try {
+            return context.parseInnerQueryBuilder();
+        } catch (IOException e) {
+            throw new ParsingException(parser.getTokenLocation(), "Failed to parse", e);
+        } finally {
+            context.reset(null);
+        }
+    }
+
     @Override
     public Iterator<Mapper> iterator() {
-        return Arrays.<Mapper>asList(queryTermsField, unknownQueryField).iterator();
+        return Arrays.<Mapper>asList(queryTermsField, unknownQueryField, queryBuilderField).iterator();
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhase.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhase.java
new file mode 100644
index 00000000000..c1f9720b53b
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhase.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.percolator;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.ReaderUtil;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.index.percolator.PercolatorFieldMapper;
+import org.elasticsearch.index.percolator.PercolatorQueryCache;
+import org.elasticsearch.index.query.ParsedQuery;
+import org.elasticsearch.index.query.PercolatorQuery;
+import org.elasticsearch.search.SearchParseElement;
+import org.elasticsearch.search.fetch.FetchSubPhase;
+import org.elasticsearch.search.highlight.HighlightPhase;
+import org.elasticsearch.search.highlight.SearchContextHighlight;
+import org.elasticsearch.search.internal.InternalSearchHit;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.SubSearchContext;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+// Highlighting in the case of the percolator query is a bit different, because the PercolatorQuery itself doesn't get highlighted,
+// but the source of the PercolatorQuery gets highlighted by each hit with type '.percolator' (percolator queries).
+public class PercolatorHighlightSubFetchPhase implements FetchSubPhase {
+
+    private final HighlightPhase highlightPhase;
+
+    @Inject
+    public PercolatorHighlightSubFetchPhase(HighlightPhase highlightPhase) {
+        this.highlightPhase = highlightPhase;
+    }
+
+    @Override
+    public boolean hitsExecutionNeeded(SearchContext context) {
+        return context.highlight() != null && locatePercolatorQuery(context.query()) != null;
+    }
+
+    @Override
+    public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
+        PercolatorQuery percolatorQuery = locatePercolatorQuery(context.query());
+        if (percolatorQuery == null) {
+            // shouldn't happen as we checked for the existence of a percolator query in hitsExecutionNeeded(...)
+            throw new IllegalStateException("couldn't locate percolator query");
+        }
+
+        List<LeafReaderContext> ctxs = context.searcher().getIndexReader().leaves();
+        PercolatorQueryCache queriesRegistry = context.percolatorQueryCache();
+        IndexSearcher percolatorIndexSearcher = percolatorQuery.getPercolatorIndexSearcher();
+
+        LeafReaderContext percolatorLeafReaderContext = percolatorIndexSearcher.getIndexReader().leaves().get(0);
+        FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext();
+        SubSearchContext subSearchContext =
+                createSubSearchContext(context, percolatorLeafReaderContext, percolatorQuery.getDocumentSource());
+
+        for (InternalSearchHit hit : hits) {
+            if (PercolatorFieldMapper.TYPE_NAME.equals(hit.getType())) {
+                LeafReaderContext ctx = ctxs.get(ReaderUtil.subIndex(hit.docId(), ctxs));
+                Query query = queriesRegistry.getQueries(ctx).getQuery(hit.docId() - ctx.docBase);
+                subSearchContext.parsedQuery(new ParsedQuery(query));
+                hitContext.reset(
+                    new InternalSearchHit(0, "unknown", new Text(percolatorQuery.getDocumentType()), Collections.emptyMap()),
+                    percolatorLeafReaderContext, 0, percolatorIndexSearcher
+                );
+                hitContext.cache().clear();
+                highlightPhase.hitExecute(subSearchContext, hitContext);
+                hit.highlightFields().putAll(hitContext.hit().getHighlightFields());
+            }
+        }
+
+    }
+
+    @Override
+    public Map<String, ? extends SearchParseElement> parseElements() {
+        return Collections.emptyMap();
+    }
+
+    @Override
+    public boolean hitExecutionNeeded(SearchContext context) {
+        return false;
+    }
+
+    @Override
+    public void hitExecute(SearchContext context, HitContext hitContext) {
+    }
+
+    static PercolatorQuery locatePercolatorQuery(Query query) {
+        if (query instanceof PercolatorQuery) {
+            return (PercolatorQuery) query;
+        } else if (query instanceof BooleanQuery) {
+            for (BooleanClause clause : ((BooleanQuery) query).clauses()) {
+                PercolatorQuery result = locatePercolatorQuery(clause.getQuery());
+                if (result != null) {
+                    return result;
+                }
+            }
+        } else if (query instanceof ConstantScoreQuery) {
+            return locatePercolatorQuery(((ConstantScoreQuery) query).getQuery());
+        } else if (query instanceof BoostQuery) {
+            return locatePercolatorQuery(((BoostQuery) query).getQuery());
+        }
+
+        return null;
+    }
+
+    private SubSearchContext createSubSearchContext(SearchContext context, LeafReaderContext leafReaderContext, BytesReference source) {
+        SubSearchContext subSearchContext = new SubSearchContext(context);
+        subSearchContext.highlight(new SearchContextHighlight(context.highlight().fields()));
+        // Enforce highlighting by source, because MemoryIndex doesn't support stored fields.
+        subSearchContext.highlight().globalForceSource(true);
+        subSearchContext.lookup().source().setSegmentAndDocument(leafReaderContext, 0);
+        subSearchContext.lookup().source().setSource(source);
+        return subSearchContext;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java
deleted file mode 100644
index 67ba0aaf1d2..00000000000
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.percolator;
-
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.metrics.CounterMetric;
-import org.elasticsearch.common.metrics.MeanMetric;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.fielddata.IndexFieldDataService;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.Uid;
-import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
-import org.elasticsearch.index.mapper.internal.UidFieldMapper;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.shard.AbstractIndexShardComponent;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.percolator.PercolatorService;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Each shard will have a percolator registry even if there isn't a {@link PercolatorService#TYPE_NAME} document type in the index.
- * For shards with indices that have no {@link PercolatorService#TYPE_NAME} document type, this will hold no percolate queries.
- * <p>
- * Once a document type has been created, the real-time percolator will start to listen to write events and update the
- * this registry with queries in real time.
- */
-public final class PercolatorQueriesRegistry extends AbstractIndexShardComponent implements Closeable {
-
-    public final static Setting<Boolean> INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING = Setting.boolSetting("index.percolator.map_unmapped_fields_as_string", false, false, Setting.Scope.INDEX);
-
-    private final ConcurrentMap<BytesRef, Query> percolateQueries = ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency();
-    private final QueryShardContext queryShardContext;
-    private boolean mapUnmappedFieldsAsString;
-    private final MeanMetric percolateMetric = new MeanMetric();
-    private final CounterMetric currentMetric = new CounterMetric();
-    private final CounterMetric numberOfQueries = new CounterMetric();
-
-    public PercolatorQueriesRegistry(ShardId shardId, IndexSettings indexSettings, QueryShardContext queryShardContext) {
-        super(shardId, indexSettings);
-        this.queryShardContext = queryShardContext;
-        this.mapUnmappedFieldsAsString = indexSettings.getValue(INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING);
-    }
-
-    public ConcurrentMap<BytesRef, Query> getPercolateQueries() {
-        return percolateQueries;
-    }
-
-    @Override
-    public void close() {
-        clear();
-    }
-
-    public void clear() {
-        percolateQueries.clear();
-    }
-
-
-    public void addPercolateQuery(String idAsString, BytesReference source) {
-        Query newquery = parsePercolatorDocument(idAsString, source);
-        BytesRef id = new BytesRef(idAsString);
-        percolateQueries.put(id, newquery);
-        numberOfQueries.inc();
-
-    }
-
-    public void removePercolateQuery(String idAsString) {
-        BytesRef id = new BytesRef(idAsString);
-        Query query = percolateQueries.remove(id);
-        if (query != null) {
-            numberOfQueries.dec();
-        }
-    }
-
-    public Query parsePercolatorDocument(String id, BytesReference source) {
-        try (XContentParser sourceParser = XContentHelper.createParser(source)) {
-            String currentFieldName = null;
-            XContentParser.Token token = sourceParser.nextToken(); // move the START_OBJECT
-            if (token != XContentParser.Token.START_OBJECT) {
-                throw new ElasticsearchException("failed to parse query [" + id + "], not starting with OBJECT");
-            }
-            while ((token = sourceParser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    currentFieldName = sourceParser.currentName();
-                } else if (token == XContentParser.Token.START_OBJECT) {
-                    if ("query".equals(currentFieldName)) {
-                        return parseQuery(queryShardContext, mapUnmappedFieldsAsString, sourceParser);
-                    } else {
-                        sourceParser.skipChildren();
-                    }
-                } else if (token == XContentParser.Token.START_ARRAY) {
-                    sourceParser.skipChildren();
-                }
-            }
-        } catch (Exception e) {
-            throw new PercolatorException(shardId().getIndex(), "failed to parse query [" + id + "]", e);
-        }
-        return null;
-    }
-
-    public static Query parseQuery(QueryShardContext queryShardContext, boolean mapUnmappedFieldsAsString, XContentParser parser) {
-        QueryShardContext context = new QueryShardContext(queryShardContext);
-        try {
-            context.reset(parser);
-            // This means that fields in the query need to exist in the mapping prior to registering this query
-            // The reason that this is required, is that if a field doesn't exist then the query assumes defaults, which may be undesired.
-            //
-            // Even worse when fields mentioned in percolator queries do go added to map after the queries have been registered
-            // then the percolator queries don't work as expected any more.
-            //
-            // Query parsing can't introduce new fields in mappings (which happens when registering a percolator query),
-            // because field type can't be inferred from queries (like document do) so the best option here is to disallow
-            // the usage of unmapped fields in percolator queries to avoid unexpected behaviour
-            //
-            // if index.percolator.map_unmapped_fields_as_string is set to true, query can contain unmapped fields which will be mapped
-            // as an analyzed string.
-            context.setAllowUnmappedFields(false);
-            context.setMapUnmappedFieldAsString(mapUnmappedFieldsAsString);
-            return context.parseInnerQuery();
-        } catch (IOException e) {
-            throw new ParsingException(parser.getTokenLocation(), "Failed to parse", e);
-        } finally {
-            context.reset(null);
-        }
-    }
-
-    public void loadQueries(IndexReader reader) {
-        logger.trace("loading percolator queries...");
-        final int loadedQueries;
-        try {
-            Query query = new TermQuery(new Term(TypeFieldMapper.NAME, PercolatorService.TYPE_NAME));
-            QueriesLoaderCollector queryCollector = new QueriesLoaderCollector(PercolatorQueriesRegistry.this, logger);
-            IndexSearcher indexSearcher = new IndexSearcher(reader);
-            indexSearcher.setQueryCache(null);
-            indexSearcher.search(query, queryCollector);
-            Map<BytesRef, Query> queries = queryCollector.queries();
-            for (Map.Entry<BytesRef, Query> entry : queries.entrySet()) {
-                percolateQueries.put(entry.getKey(), entry.getValue());
-                numberOfQueries.inc();
-            }
-            loadedQueries = queries.size();
-        } catch (Exception e) {
-            throw new PercolatorException(shardId.getIndex(), "failed to load queries from percolator index", e);
-        }
-        logger.debug("done loading [{}] percolator queries", loadedQueries);
-    }
-
-    public boolean isPercolatorQuery(Engine.Index operation) {
-        if (PercolatorService.TYPE_NAME.equals(operation.type())) {
-            parsePercolatorDocument(operation.id(), operation.source());
-            return true;
-        }
-        return false;
-    }
-
-    public boolean isPercolatorQuery(Engine.Delete operation) {
-        return PercolatorService.TYPE_NAME.equals(operation.type());
-    }
-
-    public synchronized void updatePercolateQuery(Engine engine, String id) {
-        // this can be called out of order as long as for every change to a percolator document it's invoked. This will always
-        // fetch the latest change but might fetch the same change twice if updates / deletes happen concurrently.
-        try (Engine.GetResult getResult = engine.get(new Engine.Get(true, new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(PercolatorService.TYPE_NAME, id))))) {
-            if (getResult.exists()) {
-                addPercolateQuery(id, getResult.source().source);
-            } else {
-                removePercolateQuery(id);
-            }
-        }
-    }
-
-    public void prePercolate() {
-        currentMetric.inc();
-    }
-
-    public void postPercolate(long tookInNanos) {
-        currentMetric.dec();
-        percolateMetric.inc(tookInNanos);
-    }
-
-    /**
-     * @return The current metrics
-     */
-    public PercolateStats stats() {
-        return new PercolateStats(percolateMetric.count(), TimeUnit.NANOSECONDS.toMillis(percolateMetric.sum()), currentMetric.count(), -1, numberOfQueries.count());
-    }
-
-    // Enable when a more efficient manner is found for estimating the size of a Lucene query.
-    /*private static long computeSizeInMemory(HashedBytesRef id, Query query) {
-        long size = (3 * RamUsageEstimator.NUM_BYTES_INT) + RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + id.bytes.bytes.length;
-        size += RamEstimator.sizeOf(query);
-        return size;
-    }
-
-    private static final class RamEstimator {
-        // we move this into it's own class to exclude it from the forbidden API checks
-        // it's fine to use here!
-        static long sizeOf(Query query) {
-            return RamUsageEstimator.sizeOf(query);
-        }
-    }*/
-}
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java
new file mode 100644
index 00000000000..78d8af06827
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.percolator;
+
+import com.carrotsearch.hppc.IntObjectHashMap;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.cache.Cache;
+import org.elasticsearch.common.cache.CacheBuilder;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.AbstractIndexComponent;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexWarmer;
+import org.elasticsearch.index.IndexWarmer.TerminationHandle;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.engine.Engine.Searcher;
+import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
+import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
+import org.elasticsearch.index.query.PercolatorQuery;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.shard.ShardUtils;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.function.Supplier;
+
+public final class PercolatorQueryCache extends AbstractIndexComponent
+        implements Closeable, LeafReader.CoreClosedListener, PercolatorQuery.QueryRegistry {
+
+    public final static Setting<Boolean> INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING =
+            Setting.boolSetting("index.percolator.map_unmapped_fields_as_string", false, Setting.Property.IndexScope);
+
+    public final static XContentType QUERY_BUILDER_CONTENT_TYPE = XContentType.SMILE;
+
+    private final Supplier<QueryShardContext> queryShardContextSupplier;
+    private final Cache<Object, QueriesLeaf> cache;
+    private final boolean mapUnmappedFieldsAsString;
+
+    public PercolatorQueryCache(IndexSettings indexSettings, Supplier<QueryShardContext> queryShardContextSupplier) {
+        super(indexSettings);
+        this.queryShardContextSupplier = queryShardContextSupplier;
+        cache = CacheBuilder.<Object, QueriesLeaf>builder().build();
+        this.mapUnmappedFieldsAsString = indexSettings.getValue(INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING);
+    }
+
+    @Override
+    public Leaf getQueries(LeafReaderContext ctx) {
+        QueriesLeaf percolatorQueries = cache.get(ctx.reader().getCoreCacheKey());
+        if (percolatorQueries == null) {
+            throw new IllegalStateException("queries not loaded, queries should be have been preloaded during index warming...");
+        }
+        return percolatorQueries;
+    }
+
+    public IndexWarmer.Listener createListener(ThreadPool threadPool) {
+        return new IndexWarmer.Listener() {
+
+            final Executor executor = threadPool.executor(ThreadPool.Names.WARMER);
+
+            @Override
+            public TerminationHandle warmReader(IndexShard indexShard, Searcher searcher) {
+                final CountDownLatch latch = new CountDownLatch(searcher.reader().leaves().size());
+                for (final LeafReaderContext ctx : searcher.reader().leaves()) {
+                    if (cache.get(ctx.reader().getCoreCacheKey()) != null) {
+                        latch.countDown();
+                        continue;
+                    }
+                    executor.execute(() -> {
+                        try {
+                            final long start = System.nanoTime();
+                            QueriesLeaf queries = loadQueries(ctx, indexShard.indexSettings().getIndexVersionCreated());
+                            cache.put(ctx.reader().getCoreCacheKey(), queries);
+                            if (indexShard.warmerService().logger().isTraceEnabled()) {
+                                indexShard.warmerService().logger().trace(
+                                        "loading percolator queries took [{}]",
+                                        TimeValue.timeValueNanos(System.nanoTime() - start)
+                                );
+                            }
+                        } catch (Throwable t) {
+                            indexShard.warmerService().logger().warn("failed to load percolator queries", t);
+                        } finally {
+                            latch.countDown();
+                        }
+                    });
+                }
+                return () -> latch.await();
+            }
+        };
+    }
+
+    QueriesLeaf loadQueries(LeafReaderContext context, Version indexVersionCreated) throws IOException {
+        LeafReader leafReader = context.reader();
+        ShardId shardId = ShardUtils.extractShardId(leafReader);
+        if (shardId == null) {
+            throw new IllegalStateException("can't resolve shard id");
+        }
+        if (indexSettings.getIndex().equals(shardId.getIndex()) == false) {
+            // percolator cache insanity
+            String message = "Trying to load queries for index " + shardId.getIndex() + " with cache of index " + indexSettings.getIndex();
+            throw new IllegalStateException(message);
+        }
+
+        IntObjectHashMap<Query> queries = new IntObjectHashMap<>();
+        boolean legacyLoading = indexVersionCreated.before(Version.V_5_0_0_alpha1);
+        PostingsEnum postings = leafReader.postings(new Term(TypeFieldMapper.NAME, PercolatorFieldMapper.TYPE_NAME), PostingsEnum.NONE);
+        if (postings != null) {
+            if (legacyLoading) {
+                LegacyQueryFieldVisitor visitor = new LegacyQueryFieldVisitor();
+                for (int docId = postings.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = postings.nextDoc()) {
+                    leafReader.document(docId, visitor);
+                    queries.put(docId, parseLegacyPercolatorDocument(docId, visitor.source));
+                    visitor.source = null; // reset
+                }
+            } else {
+                BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues(PercolatorFieldMapper.QUERY_BUILDER_FULL_FIELD_NAME);
+                if (binaryDocValues != null) {
+                    for (int docId = postings.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = postings.nextDoc()) {
+                        BytesRef queryBuilder = binaryDocValues.get(docId);
+                        if (queryBuilder.length > 0) {
+                            queries.put(docId, parseQueryBuilder(docId, queryBuilder));
+                        }
+                    }
+                }
+            }
+        }
+        leafReader.addCoreClosedListener(this);
+        return new QueriesLeaf(shardId, queries);
+    }
+
+    private Query parseQueryBuilder(int docId, BytesRef queryBuilder) {
+        XContent xContent = QUERY_BUILDER_CONTENT_TYPE.xContent();
+        try (XContentParser sourceParser = xContent.createParser(queryBuilder.bytes, queryBuilder.offset, queryBuilder.length)) {
+            QueryShardContext context = queryShardContextSupplier.get();
+            return PercolatorFieldMapper.parseQuery(context, mapUnmappedFieldsAsString, sourceParser);
+        } catch (IOException e) {
+            throw new PercolatorException(index(), "failed to parse query builder for document  [" + docId   + "]", e);
+        }
+    }
+
+    private Query parseLegacyPercolatorDocument(int docId, BytesReference source) {
+        try (XContentParser sourceParser = XContentHelper.createParser(source)) {
+            String currentFieldName = null;
+            XContentParser.Token token = sourceParser.nextToken(); // move the START_OBJECT
+            if (token != XContentParser.Token.START_OBJECT) {
+                throw new ElasticsearchException("failed to parse query [" + docId + "], not starting with OBJECT");
+            }
+            while ((token = sourceParser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = sourceParser.currentName();
+                } else if (token == XContentParser.Token.START_OBJECT) {
+                    if ("query".equals(currentFieldName)) {
+                        QueryShardContext context = queryShardContextSupplier.get();
+                        return PercolatorFieldMapper.parseQuery(context, mapUnmappedFieldsAsString, sourceParser);
+                    } else {
+                        sourceParser.skipChildren();
+                    }
+                } else if (token == XContentParser.Token.START_ARRAY) {
+                    sourceParser.skipChildren();
+                }
+            }
+        } catch (Exception e) {
+            throw new PercolatorException(index(), "failed to parse query [" + docId + "]", e);
+        }
+        return null;
+    }
+
+    public PercolatorQueryCacheStats getStats(ShardId shardId) {
+        int numberOfQueries = 0;
+        for (QueriesLeaf queries : cache.values()) {
+            if (shardId.equals(queries.shardId)) {
+                numberOfQueries += queries.queries.size();
+            }
+        }
+        return new PercolatorQueryCacheStats(numberOfQueries);
+    }
+
+    @Override
+    public void onClose(Object cacheKey) throws IOException {
+        cache.invalidate(cacheKey);
+    }
+
+    @Override
+    public void close() throws IOException {
+        cache.invalidateAll();
+    }
+
+    final static class LegacyQueryFieldVisitor extends StoredFieldVisitor {
+
+        private BytesArray source;
+
+        @Override
+        public void binaryField(FieldInfo fieldInfo, byte[] bytes) throws IOException {
+            source = new BytesArray(bytes);
+        }
+
+        @Override
+        public Status needsField(FieldInfo fieldInfo) throws IOException {
+            if (source != null)  {
+                return Status.STOP;
+            }
+            if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
+                return Status.YES;
+            } else {
+                return Status.NO;
+            }
+        }
+
+    }
+
+    final static class QueriesLeaf implements Leaf {
+
+        final ShardId shardId;
+        final IntObjectHashMap<Query> queries;
+
+        QueriesLeaf(ShardId shardId, IntObjectHashMap<Query> queries) {
+            this.shardId = shardId;
+            this.queries = queries;
+        }
+
+        @Override
+        public Query getQuery(int docId) {
+            return queries.get(docId);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCacheStats.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCacheStats.java
new file mode 100644
index 00000000000..a8e3b7f4799
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCacheStats.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.percolator;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentBuilderString;
+
+import java.io.IOException;
+
+/**
+ * Exposes percolator query cache statistics.
+ */
+public class PercolatorQueryCacheStats implements Streamable, ToXContent {
+
+    private long numQueries;
+
+    /**
+     * Noop constructor for serialization purposes.
+     */
+    public PercolatorQueryCacheStats() {
+    }
+
+    PercolatorQueryCacheStats(long numQueries) {
+        this.numQueries = numQueries;
+    }
+
+    /**
+     * @return The total number of loaded percolate queries.
+     */
+    public long getNumQueries() {
+        return numQueries;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(Fields.PERCOLATOR);
+        builder.field(Fields.QUERIES, getNumQueries());
+        builder.endObject();
+        return builder;
+    }
+
+    public void add(PercolatorQueryCacheStats percolate) {
+        if (percolate == null) {
+            return;
+        }
+
+        numQueries += percolate.getNumQueries();
+    }
+
+    static final class Fields {
+        static final XContentBuilderString PERCOLATOR = new XContentBuilderString("percolator");
+        static final XContentBuilderString QUERIES = new XContentBuilderString("num_queries");
+    }
+
+    public static PercolatorQueryCacheStats readPercolateStats(StreamInput in) throws IOException {
+        PercolatorQueryCacheStats stats = new PercolatorQueryCacheStats();
+        stats.readFrom(in);
+        return stats;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        numQueries = in.readVLong();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(numQueries);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/QueriesLoaderCollector.java b/core/src/main/java/org/elasticsearch/index/percolator/QueriesLoaderCollector.java
deleted file mode 100644
index 1bea43e4ea1..00000000000
--- a/core/src/main/java/org/elasticsearch/index/percolator/QueriesLoaderCollector.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.percolator;
-
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.SimpleCollector;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.IndexFieldDataService;
-import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
-import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.Uid;
-import org.elasticsearch.index.mapper.internal.UidFieldMapper;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- */
-final class QueriesLoaderCollector extends SimpleCollector {
-
-    private final Map<BytesRef, Query> queries = new HashMap<>();
-    private final FieldsVisitor fieldsVisitor = new FieldsVisitor(true);
-    private final PercolatorQueriesRegistry percolator;
-    private final ESLogger logger;
-
-    private LeafReader reader;
-
-    QueriesLoaderCollector(PercolatorQueriesRegistry percolator, ESLogger logger) {
-        this.percolator = percolator;
-        this.logger = logger;
-    }
-
-    public Map<BytesRef, Query> queries() {
-        return this.queries;
-    }
-
-    @Override
-    public void collect(int doc) throws IOException {
-        fieldsVisitor.reset();
-        reader.document(doc, fieldsVisitor);
-        final Uid uid = fieldsVisitor.uid();
-
-        try {
-            // id is only used for logging, if we fail we log the id in the catch statement
-            final Query parseQuery = percolator.parsePercolatorDocument(null, fieldsVisitor.source());
-            if (parseQuery != null) {
-                queries.put(new BytesRef(uid.id()), parseQuery);
-            } else {
-                logger.warn("failed to add query [{}] - parser returned null", uid);
-            }
-
-        } catch (Exception e) {
-            logger.warn("failed to add query [{}]", e, uid);
-        }
-    }
-
-    @Override
-    protected void doSetNextReader(LeafReaderContext context) throws IOException {
-        reader = context.reader();
-    }
-
-    @Override
-    public void setScorer(Scorer scorer) throws IOException {
-    }
-
-    @Override
-    public boolean needsScores() {
-        return false;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryParser.java
index 1ad64c42135..318a0b33805 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryParser.java
@@ -42,7 +42,7 @@ public class ConstantScoreQueryParser implements QueryParser<ConstantScoreQueryB
     public ConstantScoreQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
-        QueryBuilder query = null;
+        QueryBuilder<?> query = null;
         boolean queryFound = false;
         String queryName = null;
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
@@ -56,6 +56,10 @@ public class ConstantScoreQueryParser implements QueryParser<ConstantScoreQueryB
                 // skip
             } else if (token == XContentParser.Token.START_OBJECT) {
                 if (parseContext.parseFieldMatcher().match(currentFieldName, INNER_QUERY_FIELD)) {
+                    if (queryFound) {
+                        throw new ParsingException(parser.getTokenLocation(), "[" + ConstantScoreQueryBuilder.NAME + "]"
+                                + " accepts only one 'filter' element.");
+                    }
                     query = parseContext.parseInnerQueryBuilder();
                     queryFound = true;
                 } else {
@@ -69,6 +73,8 @@ public class ConstantScoreQueryParser implements QueryParser<ConstantScoreQueryB
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[constant_score] query does not support [" + currentFieldName + "]");
                 }
+            } else {
+                throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + "]");
             }
         }
         if (!queryFound) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
index ba25d99ed36..79e38fe7be9 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
@@ -23,18 +23,16 @@ import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermRangeQuery;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper;
-import org.elasticsearch.index.mapper.object.ObjectMapper;
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Objects;
 
 /**
@@ -82,38 +80,18 @@ public class ExistsQueryBuilder extends AbstractQueryBuilder<ExistsQueryBuilder>
             return Queries.newMatchNoDocsQuery();
         }
 
-        ObjectMapper objectMapper = context.getObjectMapper(fieldPattern);
-        if (objectMapper != null) {
-            // automatic make the object mapper pattern
-            fieldPattern = fieldPattern + ".*";
-        }
-
-        Collection<String> fields = context.simpleMatchToIndexNames(fieldPattern);
-        if (fields.isEmpty()) {
-            // no fields exists, so we should not match anything
-            return Queries.newMatchNoDocsQuery();
+        final Collection<String> fields;
+        if (context.getObjectMapper(fieldPattern) != null) {
+            // the _field_names field also indexes objects, so we don't have to
+            // do any more work to support exists queries on whole objects
+            fields = Collections.singleton(fieldPattern);
+        } else {
+            fields = context.simpleMatchToIndexNames(fieldPattern);
         }
 
         BooleanQuery.Builder boolFilterBuilder = new BooleanQuery.Builder();
         for (String field : fields) {
-            MappedFieldType fieldType = context.fieldMapper(field);
-            Query filter = null;
-            if (fieldNamesFieldType.isEnabled()) {
-                final String f;
-                if (fieldType != null) {
-                    f = fieldType.name();
-                } else {
-                    f = field;
-                }
-                filter = fieldNamesFieldType.termQuery(f, context);
-            }
-            // if _field_names are not indexed, we need to go the slow way
-            if (filter == null && fieldType != null) {
-                filter = fieldType.rangeQuery(null, null, true, true);
-            }
-            if (filter == null) {
-                filter = new TermRangeQuery(field, null, null, true, true);
-            }
+            Query filter = fieldNamesFieldType.termQuery(field, context);
             boolFilterBuilder.add(filter, BooleanClause.Occur.SHOULD);
         }
         return new ConstantScoreQuery(boolFilterBuilder.build());
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
index 05c2a74bb9f..2c906dc7cb1 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
@@ -250,7 +250,8 @@ public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder<GeoBounding
 
         GeoPoint luceneTopLeft = new GeoPoint(topLeft);
         GeoPoint luceneBottomRight = new GeoPoint(bottomRight);
-        if (GeoValidationMethod.isCoerce(validationMethod)) {
+        final Version indexVersionCreated = context.indexVersionCreated();
+        if (indexVersionCreated.onOrAfter(Version.V_2_2_0) || GeoValidationMethod.isCoerce(validationMethod)) {
             // Special case: if the difference between the left and right is 360 and the right is greater than the left, we are asking for
             // the complete longitude range so need to set longitude to the complete longitude range
             double right = luceneBottomRight.getLon();
@@ -265,7 +266,6 @@ public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder<GeoBounding
             }
         }
 
-        final Version indexVersionCreated = context.indexVersionCreated();
         if (indexVersionCreated.onOrAfter(Version.V_2_2_0)) {
             // if index created V_2_2 use (soon to be legacy) numeric encoding postings format
             // if index created V_2_3 > use prefix encoded postings format
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
index 784c924efcf..b11b57df175 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
@@ -219,18 +219,18 @@ public class GeoDistanceQueryBuilder extends AbstractQueryBuilder<GeoDistanceQue
             throw new QueryShardException(shardContext, "field [" + fieldName + "] is not a geo_point field");
         }
 
+        final Version indexVersionCreated = shardContext.indexVersionCreated();
         QueryValidationException exception = checkLatLon(shardContext.indexVersionCreated().before(Version.V_2_0_0));
         if (exception != null) {
             throw new QueryShardException(shardContext, "couldn't validate latitude/ longitude values", exception);
         }
 
-        if (GeoValidationMethod.isCoerce(validationMethod)) {
+        if (indexVersionCreated.onOrAfter(Version.V_2_2_0) || GeoValidationMethod.isCoerce(validationMethod)) {
             GeoUtils.normalizePoint(center, true, true);
         }
 
         double normDistance = geoDistance.normalize(this.distance, DistanceUnit.DEFAULT);
 
-        final Version indexVersionCreated = shardContext.indexVersionCreated();
         if (indexVersionCreated.before(Version.V_2_2_0)) {
             GeoPointFieldMapperLegacy.GeoPointFieldType geoFieldType = ((GeoPointFieldMapperLegacy.GeoPointFieldType) fieldType);
             IndexGeoPointFieldData indexFieldData = shardContext.getForField(fieldType);
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryParser.java
index c35a31f8d84..3828f786903 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryParser.java
@@ -120,9 +120,6 @@ public class GeoDistanceQueryParser implements QueryParser<GeoDistanceQueryBuild
                 } else if (currentFieldName.endsWith(GeoPointFieldMapper.Names.LON_SUFFIX)) {
                     point.resetLon(parser.doubleValue());
                     fieldName = currentFieldName.substring(0, currentFieldName.length() - GeoPointFieldMapper.Names.LON_SUFFIX.length());
-                } else if (currentFieldName.endsWith(GeoPointFieldMapper.Names.GEOHASH_SUFFIX)) {
-                    point.resetFromGeoHash(parser.text());
-                    fieldName = currentFieldName.substring(0, currentFieldName.length() - GeoPointFieldMapper.Names.GEOHASH_SUFFIX.length());
                 } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
                     queryName = parser.text();
                 } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
index e7b3dca7051..689acd5f86b 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
@@ -236,7 +236,7 @@ public class GeoDistanceRangeQueryBuilder extends AbstractQueryBuilder<GeoDistan
         }
 
         GeoPoint point = new GeoPoint(this.point);
-        if (GeoValidationMethod.isCoerce(validationMethod)) {
+        if (indexCreatedBeforeV2_2 == false || GeoValidationMethod.isCoerce(validationMethod)) {
             GeoUtils.normalizePoint(point, true, true);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryParser.java
index 8558a7b82a7..77799fa6278 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryParser.java
@@ -196,15 +196,6 @@ public class GeoDistanceRangeQueryParser implements QueryParser<GeoDistanceRange
                         point = new GeoPoint();
                     }
                     point.resetLon(parser.doubleValue());
-                } else if (currentFieldName.endsWith(GeoPointFieldMapper.Names.GEOHASH_SUFFIX)) {
-                    String maybeFieldName = currentFieldName.substring(0, currentFieldName.length() - GeoPointFieldMapper.Names.GEOHASH_SUFFIX.length());
-                    if (fieldName == null || fieldName.equals(maybeFieldName)) {
-                        fieldName = maybeFieldName;
-                    } else {
-                        throw new ParsingException(parser.getTokenLocation(), "[" + GeoDistanceRangeQueryBuilder.NAME +
-                                "] field name already set to [" + fieldName + "] but found [" + currentFieldName + "]");
-                    }
-                    point = GeoPoint.fromGeohash(parser.text());
                 } else if (parseContext.parseFieldMatcher().match(currentFieldName, NAME_FIELD)) {
                     queryName = parser.text();
                 } else if (parseContext.parseFieldMatcher().match(currentFieldName, BOOST_FIELD)) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
index 53fab5a3f4a..b251728a8a7 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
@@ -131,13 +131,13 @@ public class GeoPolygonQueryBuilder extends AbstractQueryBuilder<GeoPolygonQuery
             }
         }
 
-        if (GeoValidationMethod.isCoerce(validationMethod)) {
+        final Version indexVersionCreated = context.indexVersionCreated();
+        if (indexVersionCreated.onOrAfter(Version.V_2_2_0) || GeoValidationMethod.isCoerce(validationMethod)) {
             for (GeoPoint point : shell) {
                 GeoUtils.normalizePoint(point, true, true);
             }
         }
 
-        final Version indexVersionCreated = context.indexVersionCreated();
         if (indexVersionCreated.before(Version.V_2_2_0)) {
             IndexGeoPointFieldData indexFieldData = context.getForField(fieldType);
             return new GeoPolygonQuery(indexFieldData, shell.toArray(new GeoPoint[shellSize]));
diff --git a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
index 3c5ca1ce444..324552d7b54 100644
--- a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
@@ -189,7 +189,7 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
         builder.field(HasChildQueryParser.QUERY_FIELD.getPreferredName());
         query.toXContent(builder, params);
         builder.field(HasChildQueryParser.TYPE_FIELD.getPreferredName(), type);
-        builder.field(HasChildQueryParser.SCORE_MODE_FIELD.getPreferredName(), scoreMode.name().toLowerCase(Locale.ROOT));
+        builder.field(HasChildQueryParser.SCORE_MODE_FIELD.getPreferredName(), HasChildQueryParser.scoreModeAsString(scoreMode));
         builder.field(HasChildQueryParser.MIN_CHILDREN_FIELD.getPreferredName(), minChildren);
         builder.field(HasChildQueryParser.MAX_CHILDREN_FIELD.getPreferredName(), maxChildren);
         printBoostAndQueryName(builder);
@@ -257,12 +257,7 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
         innerQuery = Queries.filtered(innerQuery, childDocMapper.typeFilter());
 
         final ParentChildIndexFieldData parentChildIndexFieldData = context.getForField(parentFieldMapper.fieldType());
-        int maxChildren = maxChildren();
-        // 0 in pre 2.x p/c impl means unbounded
-        if (maxChildren == 0) {
-            maxChildren = Integer.MAX_VALUE;
-        }
-        return new LateParsingQuery(parentDocMapper.typeFilter(), innerQuery, minChildren(), maxChildren,
+        return new LateParsingQuery(parentDocMapper.typeFilter(), innerQuery, minChildren(), maxChildren(),
                                     parentType, scoreMode, parentChildIndexFieldData, context.getSearchSimilarity());
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java
index b3ca30f0739..7e28cd20588 100644
--- a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.support.QueryInnerHits;
 
 import java.io.IOException;
+import java.util.Locale;
 
 /**
  * A query parser for <tt>has_child</tt> queries.
@@ -104,12 +105,21 @@ public class HasChildQueryParser implements QueryParser<HasChildQueryBuilder> {
             return ScoreMode.Max;
         } else if ("avg".equals(scoreModeString)) {
             return ScoreMode.Avg;
-        } else if ("total".equals(scoreModeString)) {
+        } else if ("sum".equals(scoreModeString)) {
             return ScoreMode.Total;
         }
         throw new IllegalArgumentException("No score mode for child query [" + scoreModeString + "] found");
     }
 
+    public static String scoreModeAsString(ScoreMode scoreMode) {
+        if (scoreMode == ScoreMode.Total) {
+            // Lucene uses 'total' but 'sum' is more consistent with other elasticsearch APIs
+            return "sum";
+        } else {
+            return scoreMode.name().toLowerCase(Locale.ROOT);
+        }
+    }
+
     @Override
     public HasChildQueryBuilder getBuilderPrototype() {
         return HasChildQueryBuilder.PROTOTYPE;
diff --git a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
index 596c2499211..bd5f348db33 100644
--- a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
@@ -121,7 +121,7 @@ public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder>
         query.toXContent(builder, params);
         builder.field(NestedQueryParser.PATH_FIELD.getPreferredName(), path);
         if (scoreMode != null) {
-            builder.field(NestedQueryParser.SCORE_MODE_FIELD.getPreferredName(), scoreMode.name().toLowerCase(Locale.ROOT));
+            builder.field(NestedQueryParser.SCORE_MODE_FIELD.getPreferredName(), HasChildQueryParser.scoreModeAsString(scoreMode));
         }
         printBoostAndQueryName(builder);
         if (queryInnerHits != null) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java
index ba5d7c2447e..218919f7ed2 100644
--- a/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java
@@ -68,20 +68,7 @@ public class NestedQueryParser implements QueryParser<NestedQueryBuilder> {
                 } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
                     boost = parser.floatValue();
                 } else if (parseContext.parseFieldMatcher().match(currentFieldName, SCORE_MODE_FIELD)) {
-                    String sScoreMode = parser.text();
-                    if ("avg".equals(sScoreMode)) {
-                        scoreMode = ScoreMode.Avg;
-                    } else if ("min".equals(sScoreMode)) {
-                        scoreMode = ScoreMode.Min;
-                    } else if ("max".equals(sScoreMode)) {
-                        scoreMode = ScoreMode.Max;
-                    } else if ("total".equals(sScoreMode) || "sum".equals(sScoreMode)) {
-                        scoreMode = ScoreMode.Total;
-                    } else if ("none".equals(sScoreMode)) {
-                        scoreMode = ScoreMode.None;
-                    } else {
-                        throw new ParsingException(parser.getTokenLocation(), "illegal score_mode for nested query [" + sScoreMode + "]");
-                    }
+                    scoreMode = HasChildQueryParser.parseScoreMode(parser.text());
                 } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
                     queryName = parser.text();
                 } else {
diff --git a/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java
index f9bd7623f35..4a2efa95c9a 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java
@@ -19,13 +19,18 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.DocValuesTermsQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
+import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -71,7 +76,12 @@ public final class ParentIdQueryBuilder extends AbstractQueryBuilder<ParentIdQue
             throw new QueryShardException(context, "[" + NAME + "] _parent field has no parent type configured");
         }
         String fieldName = ParentFieldMapper.joinField(parentFieldMapper.type());
-        return new DocValuesTermsQuery(fieldName, id);
+
+        BooleanQuery.Builder query = new BooleanQuery.Builder();
+        query.add(new DocValuesTermsQuery(fieldName, id), BooleanClause.Occur.MUST);
+        // Need to take child type into account, otherwise a child doc of different type with the same id could match
+        query.add(new TermQuery(new Term(TypeFieldMapper.NAME, type)), BooleanClause.Occur.FILTER);
+        return query.build();
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/percolator/PercolatorQuery.java b/core/src/main/java/org/elasticsearch/index/query/PercolatorQuery.java
similarity index 60%
rename from core/src/main/java/org/elasticsearch/percolator/PercolatorQuery.java
rename to core/src/main/java/org/elasticsearch/index/query/PercolatorQuery.java
index 98be7d308af..0b22b17f65e 100644
--- a/core/src/main/java/org/elasticsearch/percolator/PercolatorQuery.java
+++ b/core/src/main/java/org/elasticsearch/index/query/PercolatorQuery.java
@@ -17,10 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.percolator;
+package org.elasticsearch.index.query;
 
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanQuery;
@@ -31,51 +30,48 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
-import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Accountable;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.index.fieldvisitor.SingleFieldsVisitor;
-import org.elasticsearch.index.mapper.internal.UidFieldMapper;
 import org.elasticsearch.index.percolator.ExtractQueryTermsService;
 
 import java.io.IOException;
-import java.util.Map;
+import java.util.Collection;
+import java.util.Objects;
 import java.util.Set;
 
 import static org.apache.lucene.search.BooleanClause.Occur.FILTER;
 import static org.apache.lucene.search.BooleanClause.Occur.MUST;
 
-final class PercolatorQuery extends Query {
+public final class PercolatorQuery extends Query implements Accountable {
 
-    public static final float MATCH_COST =
-            (1 << 14) // stored field access cost, approximated by the number of bytes in a block
-            + 1000;   // cost of matching the query against the document, arbitrary as it would be really complex to estimate
+    // cost of matching the query against the document, arbitrary as it would be really complex to estimate
+    public static final float MATCH_COST = 1000;
 
-    static class Builder {
+    public static class Builder {
 
+        private final String docType;
+        private final QueryRegistry queryRegistry;
+        private final BytesReference documentSource;
         private final IndexSearcher percolatorIndexSearcher;
-        private final Map<BytesRef, Query> percolatorQueries;
 
-        private Query percolateQuery;
         private Query queriesMetaDataQuery;
         private final Query percolateTypeQuery;
 
         /**
-         * @param percolatorIndexSearcher The index searcher on top of the in-memory index that holds the document being percolated
-         * @param percolatorQueries All the registered percolator queries
-         * @param percolateTypeQuery A query that identifies all document containing percolator queries
+         * @param docType                   The type of the document being percolated
+         * @param queryRegistry             The registry holding all the percolator queries as Lucene queries.
+         * @param documentSource            The source of the document being percolated
+         * @param percolatorIndexSearcher   The index searcher on top of the in-memory index that holds the document being percolated
+         * @param percolateTypeQuery        A query that identifies all document containing percolator queries
          */
-        Builder(IndexSearcher percolatorIndexSearcher, Map<BytesRef, Query> percolatorQueries, Query percolateTypeQuery) {
-            this.percolatorIndexSearcher = percolatorIndexSearcher;
-            this.percolatorQueries = percolatorQueries;
-            this.percolateTypeQuery = percolateTypeQuery;
-        }
-
-        /**
-         * Optionally sets a query that reduces the number of queries to percolate based on custom metadata attached
-         * on the percolator documents.
-         */
-        void setPercolateQuery(Query percolateQuery) {
-            this.percolateQuery = percolateQuery;
+        public Builder(String docType, QueryRegistry queryRegistry, BytesReference documentSource, IndexSearcher percolatorIndexSearcher,
+                Query percolateTypeQuery) {
+            this.docType = Objects.requireNonNull(docType);
+            this.documentSource = Objects.requireNonNull(documentSource);
+            this.percolatorIndexSearcher = Objects.requireNonNull(percolatorIndexSearcher);
+            this.queryRegistry = Objects.requireNonNull(queryRegistry);
+            this.percolateTypeQuery = Objects.requireNonNull(percolateTypeQuery);
         }
 
         /**
@@ -85,39 +81,43 @@ final class PercolatorQuery extends Query {
          * @param extractedTermsFieldName The name of the field to get the extracted terms from
          * @param unknownQueryFieldname The field used to mark documents whose queries couldn't all get extracted
          */
-        void extractQueryTermsQuery(String extractedTermsFieldName, String unknownQueryFieldname) throws IOException {
-            this.queriesMetaDataQuery = ExtractQueryTermsService.createQueryTermsQuery(percolatorIndexSearcher.getIndexReader(), extractedTermsFieldName, unknownQueryFieldname);
+        public void extractQueryTermsQuery(String extractedTermsFieldName, String unknownQueryFieldname) throws IOException {
+            this.queriesMetaDataQuery = ExtractQueryTermsService.createQueryTermsQuery(
+                    percolatorIndexSearcher.getIndexReader(), extractedTermsFieldName, unknownQueryFieldname
+            );
         }
 
-        PercolatorQuery build() {
+        public PercolatorQuery build() {
             BooleanQuery.Builder builder = new BooleanQuery.Builder();
             builder.add(percolateTypeQuery, FILTER);
             if (queriesMetaDataQuery != null) {
                 builder.add(queriesMetaDataQuery, FILTER);
             }
-            if (percolateQuery != null){
-                builder.add(percolateQuery, MUST);
-            }
-            return new PercolatorQuery(builder.build(), percolatorIndexSearcher, percolatorQueries);
+            return new PercolatorQuery(docType, queryRegistry, documentSource, builder.build(), percolatorIndexSearcher);
         }
 
     }
 
+    private final String documentType;
+    private final QueryRegistry queryRegistry;
+    private final BytesReference documentSource;
     private final Query percolatorQueriesQuery;
     private final IndexSearcher percolatorIndexSearcher;
-    private final Map<BytesRef, Query> percolatorQueries;
 
-    private PercolatorQuery(Query percolatorQueriesQuery, IndexSearcher percolatorIndexSearcher, Map<BytesRef, Query> percolatorQueries) {
+    private PercolatorQuery(String documentType, QueryRegistry queryRegistry, BytesReference documentSource,
+                            Query percolatorQueriesQuery, IndexSearcher percolatorIndexSearcher) {
+        this.documentType = documentType;
+        this.documentSource = documentSource;
         this.percolatorQueriesQuery = percolatorQueriesQuery;
+        this.queryRegistry = queryRegistry;
         this.percolatorIndexSearcher = percolatorIndexSearcher;
-        this.percolatorQueries = percolatorQueries;
     }
 
     @Override
     public Query rewrite(IndexReader reader) throws IOException {
         Query rewritten = percolatorQueriesQuery.rewrite(reader);
         if (rewritten != percolatorQueriesQuery) {
-            return new PercolatorQuery(rewritten, percolatorIndexSearcher, percolatorQueries);
+            return new PercolatorQuery(documentType, queryRegistry, documentSource, rewritten, percolatorIndexSearcher);
         } else {
             return this;
         }
@@ -160,7 +160,7 @@ final class PercolatorQuery extends Query {
                     return null;
                 }
 
-                final LeafReader leafReader = leafReaderContext.reader();
+                final QueryRegistry.Leaf percolatorQueries = queryRegistry.getQueries(leafReaderContext);
                 return new Scorer(this) {
 
                     @Override
@@ -173,7 +173,7 @@ final class PercolatorQuery extends Query {
                         return new TwoPhaseIterator(approximation.iterator()) {
                             @Override
                             public boolean matches() throws IOException {
-                                return matchDocId(approximation.docID(), leafReader);
+                                return matchDocId(approximation.docID());
                             }
 
                             @Override
@@ -198,27 +198,30 @@ final class PercolatorQuery extends Query {
                         return approximation.docID();
                     }
 
-                    boolean matchDocId(int docId, LeafReader leafReader) throws IOException {
-                        SingleFieldsVisitor singleFieldsVisitor = new SingleFieldsVisitor(UidFieldMapper.NAME);
-                        leafReader.document(docId, singleFieldsVisitor);
-                        BytesRef percolatorQueryId = new BytesRef(singleFieldsVisitor.uid().id());
-                        return matchQuery(percolatorQueryId);
+                    boolean matchDocId(int docId) throws IOException {
+                        Query query = percolatorQueries.getQuery(docId);
+                        if (query != null) {
+                            return Lucene.exists(percolatorIndexSearcher, query);
+                        } else {
+                            return false;
+                        }
                     }
                 };
             }
         };
     }
 
-    boolean matchQuery(BytesRef percolatorQueryId) throws IOException {
-        Query percolatorQuery = percolatorQueries.get(percolatorQueryId);
-        if (percolatorQuery != null) {
-            return Lucene.exists(percolatorIndexSearcher, percolatorQuery);
-        } else {
-            return false;
-        }
+    public IndexSearcher getPercolatorIndexSearcher() {
+        return percolatorIndexSearcher;
     }
 
-    private final Object instance = new Object();
+    public String getDocumentType() {
+        return documentType;
+    }
+
+    public BytesReference getDocumentSource() {
+        return documentSource;
+    }
 
     @Override
     public boolean equals(Object o) {
@@ -228,19 +231,46 @@ final class PercolatorQuery extends Query {
 
         PercolatorQuery that = (PercolatorQuery) o;
 
-        return instance.equals(that.instance);
+        if (!documentType.equals(that.documentType)) return false;
+        return documentSource.equals(that.documentSource);
 
     }
 
     @Override
     public int hashCode() {
         int result = super.hashCode();
-        result = 31 * result + instance.hashCode();
+        result = 31 * result + documentType.hashCode();
+        result = 31 * result + documentSource.hashCode();
         return result;
     }
 
     @Override
     public String toString(String s) {
-        return "PercolatorQuery{inner={" + percolatorQueriesQuery.toString(s)  + "}}";
+        return "PercolatorQuery{document_type={" + documentType + "},document_source={" + documentSource.toUtf8() +
+                "},inner={" + percolatorQueriesQuery.toString(s)  + "}}";
     }
+
+    @Override
+    public long ramBytesUsed() {
+        long sizeInBytes = 0;
+        if (documentSource.hasArray()) {
+            sizeInBytes += documentSource.array().length;
+        } else {
+            sizeInBytes += documentSource.length();
+        }
+        return sizeInBytes;
+    }
+
+    public interface QueryRegistry {
+
+        Leaf getQueries(LeafReaderContext ctx);
+
+        interface Leaf {
+
+            Query getQuery(int docId);
+
+        }
+
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryBuilder.java
new file mode 100644
index 00000000000..7ca70eef6aa
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryBuilder.java
@@ -0,0 +1,375 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.query;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.MultiReader;
+import org.apache.lucene.index.SlowCompositeReaderWrapper;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.memory.MemoryIndex;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.Weight;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.DocumentMapperForType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
+import org.elasticsearch.index.mapper.internal.UidFieldMapper;
+import org.elasticsearch.index.percolator.PercolatorFieldMapper;
+import org.elasticsearch.index.percolator.PercolatorQueryCache;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.index.mapper.SourceToParse.source;
+
+public class PercolatorQueryBuilder extends AbstractQueryBuilder<PercolatorQueryBuilder> {
+
+    public static final String NAME = "percolator";
+    static final PercolatorQueryBuilder PROTO = new PercolatorQueryBuilder(null, null, null, null, null, null, null, null);
+
+    private final String documentType;
+    private final BytesReference document;
+
+    private final String indexedDocumentIndex;
+    private final String indexedDocumentType;
+    private final String indexedDocumentId;
+    private final String indexedDocumentRouting;
+    private final String indexedDocumentPreference;
+    private final Long indexedDocumentVersion;
+
+    public PercolatorQueryBuilder(String documentType, BytesReference document) {
+        if (documentType == null) {
+            throw new IllegalArgumentException("[document_type] is a required argument");
+        }
+        if (document == null) {
+            throw new IllegalArgumentException("[document] is a required argument");
+        }
+        this.documentType = documentType;
+        this.document = document;
+        indexedDocumentIndex = null;
+        indexedDocumentType = null;
+        indexedDocumentId = null;
+        indexedDocumentRouting = null;
+        indexedDocumentPreference = null;
+        indexedDocumentVersion = null;
+    }
+
+    public PercolatorQueryBuilder(String documentType, String indexedDocumentIndex, String indexedDocumentType,
+                                  String indexedDocumentId, String indexedDocumentRouting, String indexedDocumentPreference,
+                                  Long indexedDocumentVersion) {
+        if (documentType == null) {
+            throw new IllegalArgumentException("[document_type] is a required argument");
+        }
+        if (indexedDocumentIndex == null) {
+            throw new IllegalArgumentException("[index] is a required argument");
+        }
+        if (indexedDocumentType == null) {
+            throw new IllegalArgumentException("[type] is a required argument");
+        }
+        if (indexedDocumentId == null) {
+            throw new IllegalArgumentException("[id] is a required argument");
+        }
+        this.documentType = documentType;
+        this.indexedDocumentIndex = indexedDocumentIndex;
+        this.indexedDocumentType = indexedDocumentType;
+        this.indexedDocumentId = indexedDocumentId;
+        this.indexedDocumentRouting = indexedDocumentRouting;
+        this.indexedDocumentPreference = indexedDocumentPreference;
+        this.indexedDocumentVersion = indexedDocumentVersion;
+        this.document = null;
+    }
+
+    private PercolatorQueryBuilder(String documentType, BytesReference document, String indexedDocumentIndex, String indexedDocumentType,
+                                   String indexedDocumentId, String indexedDocumentRouting, String indexedDocumentPreference,
+                                   Long indexedDocumentVersion) {
+        this.documentType = documentType;
+        this.document = document;
+        this.indexedDocumentIndex = indexedDocumentIndex;
+        this.indexedDocumentType = indexedDocumentType;
+        this.indexedDocumentId = indexedDocumentId;
+        this.indexedDocumentRouting = indexedDocumentRouting;
+        this.indexedDocumentPreference = indexedDocumentPreference;
+        this.indexedDocumentVersion = indexedDocumentVersion;
+    }
+
+    @Override
+    protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(NAME);
+        builder.field(PercolatorQueryParser.DOCUMENT_TYPE_FIELD.getPreferredName(), documentType);
+        if (document != null) {
+            XContentType contentType = XContentFactory.xContentType(document);
+            if (contentType == builder.contentType()) {
+                builder.rawField(PercolatorQueryParser.DOCUMENT_FIELD.getPreferredName(), document);
+            } else {
+                XContentParser parser = XContentFactory.xContent(contentType).createParser(document);
+                parser.nextToken();
+                builder.field(PercolatorQueryParser.DOCUMENT_FIELD.getPreferredName());
+                builder.copyCurrentStructure(parser);
+            }
+        }
+        if (indexedDocumentIndex != null || indexedDocumentType != null || indexedDocumentId != null) {
+            if (indexedDocumentIndex != null) {
+                builder.field(PercolatorQueryParser.INDEXED_DOCUMENT_FIELD_INDEX.getPreferredName(), indexedDocumentIndex);
+            }
+            if (indexedDocumentType != null) {
+                builder.field(PercolatorQueryParser.INDEXED_DOCUMENT_FIELD_TYPE.getPreferredName(), indexedDocumentType);
+            }
+            if (indexedDocumentId != null) {
+                builder.field(PercolatorQueryParser.INDEXED_DOCUMENT_FIELD_ID.getPreferredName(), indexedDocumentId);
+            }
+            if (indexedDocumentRouting != null) {
+                builder.field(PercolatorQueryParser.INDEXED_DOCUMENT_FIELD_ROUTING.getPreferredName(), indexedDocumentRouting);
+            }
+            if (indexedDocumentPreference != null) {
+                builder.field(PercolatorQueryParser.INDEXED_DOCUMENT_FIELD_PREFERENCE.getPreferredName(), indexedDocumentPreference);
+            }
+            if (indexedDocumentVersion != null) {
+                builder.field(PercolatorQueryParser.INDEXED_DOCUMENT_FIELD_VERSION.getPreferredName(), indexedDocumentVersion);
+            }
+        }
+        printBoostAndQueryName(builder);
+        builder.endObject();
+    }
+
+    @Override
+    protected PercolatorQueryBuilder doReadFrom(StreamInput in) throws IOException {
+        String docType = in.readString();
+        String documentIndex = in.readOptionalString();
+        String documentType = in.readOptionalString();
+        String documentId = in.readOptionalString();
+        String documentRouting = in.readOptionalString();
+        String documentPreference = in.readOptionalString();
+        Long documentVersion = null;
+        if (in.readBoolean()) {
+            documentVersion = in.readVLong();
+        }
+        BytesReference documentSource = null;
+        if (in.readBoolean()) {
+            documentSource = in.readBytesReference();
+        }
+        return new PercolatorQueryBuilder(docType, documentSource, documentIndex, documentType, documentId,
+                documentRouting, documentPreference, documentVersion);
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeString(documentType);
+        out.writeOptionalString(indexedDocumentIndex);
+        out.writeOptionalString(indexedDocumentType);
+        out.writeOptionalString(indexedDocumentId);
+        out.writeOptionalString(indexedDocumentRouting);
+        out.writeOptionalString(indexedDocumentPreference);
+        if (indexedDocumentVersion != null) {
+            out.writeBoolean(true);
+            out.writeVLong(indexedDocumentVersion);
+        } else {
+            out.writeBoolean(false);
+        }
+        if (document != null) {
+            out.writeBoolean(true);
+            out.writeBytesReference(document);
+        } else {
+            out.writeBoolean(false);
+        }
+    }
+
+    @Override
+    protected boolean doEquals(PercolatorQueryBuilder other) {
+        return Objects.equals(documentType, other.documentType)
+                && Objects.equals(document, other.document)
+                && Objects.equals(indexedDocumentIndex, other.indexedDocumentIndex)
+                && Objects.equals(indexedDocumentType, other.indexedDocumentType)
+                && Objects.equals(indexedDocumentId, other.indexedDocumentId);
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(documentType, document, indexedDocumentIndex, indexedDocumentType, indexedDocumentId);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryShardContext) throws IOException {
+        if (document != null) {
+            return this;
+        }
+
+        GetRequest getRequest = new GetRequest(indexedDocumentIndex, indexedDocumentType, indexedDocumentId);
+        getRequest.preference("_local");
+        getRequest.routing(indexedDocumentRouting);
+        getRequest.preference(indexedDocumentPreference);
+        if (indexedDocumentVersion != null) {
+            getRequest.version(indexedDocumentVersion);
+        }
+        GetResponse getResponse = queryShardContext.getClient().get(getRequest).actionGet();
+        if (getResponse.isExists() == false) {
+            throw new ResourceNotFoundException(
+                    "indexed document [{}/{}/{}] couldn't be found", indexedDocumentIndex, indexedDocumentType, indexedDocumentId
+            );
+        }
+        return new PercolatorQueryBuilder(documentType, getResponse.getSourceAsBytesRef());
+    }
+
+    @Override
+    protected Query doToQuery(QueryShardContext context) throws IOException {
+        if (indexedDocumentIndex != null || indexedDocumentType != null || indexedDocumentId != null) {
+            throw new IllegalStateException("query builder must be rewritten first");
+        }
+
+        if (document == null) {
+            throw new IllegalStateException("nothing to percolator");
+        }
+
+        MapperService mapperService = context.getMapperService();
+        DocumentMapperForType docMapperForType = mapperService.documentMapperWithAutoCreate(documentType);
+        DocumentMapper docMapper = docMapperForType.getDocumentMapper();
+
+        ParsedDocument doc = docMapper.parse(source(document)
+            .index(context.index().getName())
+            .id("_temp_id")
+            .type(documentType));
+
+        Analyzer defaultAnalyzer = context.getAnalysisService().defaultIndexAnalyzer();
+        final IndexSearcher docSearcher;
+        if (doc.docs().size() > 1) {
+            assert docMapper.hasNestedObjects();
+            docSearcher = createMultiDocumentSearcher(docMapper, defaultAnalyzer, doc);
+        } else {
+            // TODO: we may want to bring to MemoryIndex thread local cache back...
+            // but I'm unsure about the real benefits.
+            MemoryIndex memoryIndex = new MemoryIndex(true);
+            indexDoc(docMapper, defaultAnalyzer, doc.rootDoc(), memoryIndex);
+            docSearcher = memoryIndex.createSearcher();
+            docSearcher.setQueryCache(null);
+        }
+
+        PercolatorQueryCache registry = context.getPercolatorQueryCache();
+        if (registry == null) {
+            throw new QueryShardException(context, "no percolator query registry");
+        }
+
+        Query percolateTypeQuery = new TermQuery(new Term(TypeFieldMapper.NAME, PercolatorFieldMapper.TYPE_NAME));
+        PercolatorQuery.Builder builder = new PercolatorQuery.Builder(
+                documentType, registry, document, docSearcher, percolateTypeQuery
+        );
+        Settings indexSettings = registry.getIndexSettings().getSettings();
+        if (indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null).onOrAfter(Version.V_5_0_0_alpha1)) {
+            builder.extractQueryTermsQuery(
+                    PercolatorFieldMapper.EXTRACTED_TERMS_FULL_FIELD_NAME, PercolatorFieldMapper.UNKNOWN_QUERY_FULL_FIELD_NAME
+            );
+        }
+        return builder.build();
+    }
+
+    public String getDocumentType() {
+        return documentType;
+    }
+
+    public BytesReference getDocument() {
+        return document;
+    }
+
+    private IndexSearcher createMultiDocumentSearcher(DocumentMapper docMapper, Analyzer defaultAnalyzer, ParsedDocument doc) {
+        IndexReader[] memoryIndices = new IndexReader[doc.docs().size()];
+        List<ParseContext.Document> docs = doc.docs();
+        int rootDocIndex = docs.size() - 1;
+        assert rootDocIndex > 0;
+        for (int i = 0; i < docs.size(); i++) {
+            ParseContext.Document d = docs.get(i);
+            MemoryIndex memoryIndex = new MemoryIndex(true);
+            indexDoc(docMapper, defaultAnalyzer, d, memoryIndex);
+            memoryIndices[i] = memoryIndex.createSearcher().getIndexReader();
+        }
+        try {
+            MultiReader mReader = new MultiReader(memoryIndices, true);
+            LeafReader slowReader = SlowCompositeReaderWrapper.wrap(mReader);
+            final IndexSearcher slowSearcher = new IndexSearcher(slowReader) {
+
+                @Override
+                public Weight createNormalizedWeight(Query query, boolean needsScores) throws IOException {
+                    BooleanQuery.Builder bq = new BooleanQuery.Builder();
+                    bq.add(query, BooleanClause.Occur.MUST);
+                    bq.add(Queries.newNestedFilter(), BooleanClause.Occur.MUST_NOT);
+                    return super.createNormalizedWeight(bq.build(), needsScores);
+                }
+
+            };
+            slowSearcher.setQueryCache(null);
+            return slowSearcher;
+        } catch (IOException e) {
+            throw new ElasticsearchException("Failed to create index for percolator with nested document ", e);
+        }
+    }
+
+    private void indexDoc(DocumentMapper documentMapper, Analyzer defaultAnalyzer, ParseContext.Document document,
+                          MemoryIndex memoryIndex) {
+        for (IndexableField field : document.getFields()) {
+            if (field.fieldType().indexOptions() == IndexOptions.NONE && field.name().equals(UidFieldMapper.NAME)) {
+                continue;
+            }
+
+            Analyzer analyzer = defaultAnalyzer;
+            if (documentMapper != null && documentMapper.mappers().getMapper(field.name()) != null) {
+                analyzer =  documentMapper.mappers().indexAnalyzer();
+            }
+            try {
+                try (TokenStream tokenStream = field.tokenStream(analyzer, null)) {
+                    if (tokenStream != null) {
+                        memoryIndex.addField(field.name(), tokenStream, field.boost());
+                    }
+                }
+            } catch (IOException e) {
+                throw new ElasticsearchException("Failed to create token stream", e);
+            }
+        }
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryParser.java
new file mode 100644
index 00000000000..a559db59927
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryParser.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.query;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+
+public class PercolatorQueryParser implements QueryParser<PercolatorQueryBuilder> {
+
+    public static final ParseField DOCUMENT_FIELD = new ParseField("document");
+    public static final ParseField DOCUMENT_TYPE_FIELD = new ParseField("document_type");
+    public static final ParseField INDEXED_DOCUMENT_FIELD_INDEX = new ParseField("index");
+    public static final ParseField INDEXED_DOCUMENT_FIELD_TYPE = new ParseField("type");
+    public static final ParseField INDEXED_DOCUMENT_FIELD_ID = new ParseField("id");
+    public static final ParseField INDEXED_DOCUMENT_FIELD_ROUTING = new ParseField("routing");
+    public static final ParseField INDEXED_DOCUMENT_FIELD_PREFERENCE = new ParseField("preference");
+    public static final ParseField INDEXED_DOCUMENT_FIELD_VERSION = new ParseField("version");
+
+    @Override
+    public String[] names() {
+        return new String[]{PercolatorQueryBuilder.NAME};
+    }
+
+    @Override
+    public PercolatorQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+        XContentParser parser = parseContext.parser();
+        float boost = AbstractQueryBuilder.DEFAULT_BOOST;
+
+        String documentType = null;
+
+        String indexedDocumentIndex = null;
+        String indexedDocumentType = null;
+        String indexedDocumentId = null;
+        String indexedDocumentRouting = null;
+        String indexedDocumentPreference = null;
+        Long indexedDocumentVersion = null;
+
+        BytesReference source = null;
+
+        String queryName = null;
+        String currentFieldName = null;
+
+        XContentParser.Token token;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (parseContext.parseFieldMatcher().match(currentFieldName, DOCUMENT_FIELD)) {
+                    try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
+                        builder.copyCurrentStructure(parser);
+                        builder.flush();
+                        source = builder.bytes();
+                    }
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "[" + PercolatorQueryBuilder.NAME +
+                            "] query does not support [" + token + "]");
+                }
+            } else if (token.isValue()) {
+                if (parseContext.parseFieldMatcher().match(currentFieldName, DOCUMENT_TYPE_FIELD)) {
+                    documentType = parser.text();
+                } else if (parseContext.parseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_INDEX)) {
+                    indexedDocumentIndex = parser.text();
+                } else if (parseContext.parseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_TYPE)) {
+                    indexedDocumentType = parser.text();
+                } else if (parseContext.parseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_ID)) {
+                    indexedDocumentId = parser.text();
+                } else if (parseContext.parseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_ROUTING)) {
+                    indexedDocumentRouting = parser.text();
+                } else if (parseContext.parseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_PREFERENCE)) {
+                    indexedDocumentPreference = parser.text();
+                } else if (parseContext.parseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_VERSION)) {
+                    indexedDocumentVersion = parser.longValue();
+                } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                    boost = parser.floatValue();
+                } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                    queryName = parser.text();
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "[" + PercolatorQueryBuilder.NAME +
+                            "] query does not support [" + currentFieldName + "]");
+                }
+            } else {
+                throw new ParsingException(parser.getTokenLocation(), "[" + PercolatorQueryBuilder.NAME +
+                        "] query does not support [" + token + "]");
+            }
+        }
+
+        if (documentType == null) {
+            throw new IllegalArgumentException("[" + PercolatorQueryBuilder.NAME + "] query is missing required [" +
+                    DOCUMENT_TYPE_FIELD.getPreferredName() + "] parameter");
+        }
+
+        PercolatorQueryBuilder queryBuilder;
+        if (source != null) {
+            queryBuilder = new PercolatorQueryBuilder(documentType, source);
+        } else if (indexedDocumentId != null) {
+            queryBuilder = new PercolatorQueryBuilder(documentType, indexedDocumentIndex, indexedDocumentType,
+                    indexedDocumentId, indexedDocumentRouting, indexedDocumentPreference, indexedDocumentVersion);
+        } else {
+            throw new IllegalArgumentException("[" + PercolatorQueryBuilder.NAME + "] query, nothing to percolate");
+        }
+        queryBuilder.queryName(queryName);
+        queryBuilder.boost(boost);
+        return queryBuilder;
+    }
+
+    @Override
+    public PercolatorQueryBuilder getBuilderPrototype() {
+        return PercolatorQueryBuilder.PROTO;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java b/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
index 21c1f3ff695..f04f03fcbcd 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
@@ -832,6 +832,18 @@ public abstract class QueryBuilders {
         return new ExistsQueryBuilder(name);
     }
 
+    public static PercolatorQueryBuilder percolatorQuery(String documentType, BytesReference document) {
+        return new PercolatorQueryBuilder(documentType, document);
+    }
+
+    public static PercolatorQueryBuilder percolatorQuery(String documentType, String indexedDocumentIndex,
+                                                         String indexedDocumentType, String indexedDocumentId,
+                                                         String indexedDocumentRouting, String indexedDocumentPreference,
+                                                         Long indexedDocumentVersion) {
+        return new PercolatorQueryBuilder(documentType, indexedDocumentIndex, indexedDocumentType, indexedDocumentId,
+                indexedDocumentRouting, indexedDocumentPreference, indexedDocumentVersion);
+    }
+
     private QueryBuilders() {
 
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryParser.java b/core/src/main/java/org/elasticsearch/index/query/QueryParser.java
index 0a3f6d6147c..1226564bb3b 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryParser.java
@@ -33,7 +33,7 @@ public interface QueryParser<QB extends QueryBuilder<QB>> {
     String[] names();
 
     /**
-     * Creates a new {@link QueryBuilder} from the query held by the {@link QueryShardContext}
+     * Creates a new {@link QueryBuilder} from the query held by the {@link QueryParseContext}
      * in {@link org.elasticsearch.common.xcontent.XContent} format
      *
      * @param parseContext
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
index e057aff06b1..11164659b3f 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
@@ -20,6 +20,7 @@ package org.elasticsearch.index.query;
 
 import org.elasticsearch.client.Client;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.fieldstats.FieldStatsProvider;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.ScriptService;
 
@@ -31,6 +32,7 @@ public class QueryRewriteContext {
     protected final IndexSettings indexSettings;
     protected final IndicesQueriesRegistry indicesQueriesRegistry;
     protected final QueryParseContext parseContext;
+    protected FieldStatsProvider fieldStatsProvider;
 
     public QueryRewriteContext(IndexSettings indexSettings, ScriptService scriptService, IndicesQueriesRegistry indicesQueriesRegistry) {
         this.scriptService = scriptService;
@@ -39,6 +41,14 @@ public class QueryRewriteContext {
         this.parseContext = new QueryParseContext(indicesQueriesRegistry);
     }
 
+    public void setFieldStatsProvider(FieldStatsProvider fieldStatsProvider) {
+        this.fieldStatsProvider = fieldStatsProvider;
+    }
+
+    public FieldStatsProvider getFieldStatsProvider() {
+        return fieldStatsProvider;
+    }
+
     /**
      * Returns a clients to fetch resources from local or remove nodes.
      */
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
index 6acd5272f89..63eff82ddb0 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
@@ -44,9 +44,9 @@ import org.elasticsearch.index.mapper.ContentPath;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.core.StringFieldMapper;
 import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
+import org.elasticsearch.index.percolator.PercolatorQueryCache;
 import org.elasticsearch.index.query.support.InnerHitsQueryParserHelper;
 import org.elasticsearch.index.query.support.NestedScope;
 import org.elasticsearch.index.similarity.SimilarityService;
@@ -87,13 +87,15 @@ public class QueryShardContext extends QueryRewriteContext {
 
     private final Map<String, Query> namedQueries = new HashMap<>();
     private final MapperQueryParser queryParser = new MapperQueryParser(this);
+    private final IndicesQueriesRegistry indicesQueriesRegistry;
+    private final PercolatorQueryCache percolatorQueryCache;
     private boolean allowUnmappedFields;
     private boolean mapUnmappedFieldAsString;
     private NestedScope nestedScope;
     boolean isFilter; // pkg private for testing
 
     public QueryShardContext(IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, IndexFieldDataService indexFieldDataService, MapperService mapperService, SimilarityService similarityService, ScriptService scriptService,
-                             final IndicesQueriesRegistry indicesQueriesRegistry) {
+                             final IndicesQueriesRegistry indicesQueriesRegistry, PercolatorQueryCache percolatorQueryCache) {
         super(indexSettings, scriptService, indicesQueriesRegistry);
         this.indexSettings = indexSettings;
         this.similarityService = similarityService;
@@ -101,17 +103,20 @@ public class QueryShardContext extends QueryRewriteContext {
         this.bitsetFilterCache = bitsetFilterCache;
         this.indexFieldDataService = indexFieldDataService;
         this.allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields();
-
+        this.indicesQueriesRegistry = indicesQueriesRegistry;
+        this.percolatorQueryCache = percolatorQueryCache;
+        this.nestedScope = new NestedScope();
     }
 
     public QueryShardContext(QueryShardContext source) {
-        this(source.indexSettings, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, source.similarityService, source.scriptService, source.indicesQueriesRegistry);
+        this(source.indexSettings, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, source.similarityService, source.scriptService, source.indicesQueriesRegistry, source.percolatorQueryCache);
         this.types = source.getTypes();
     }
 
 
+    @Override
     public QueryShardContext clone() {
-        return new QueryShardContext(indexSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry);
+        return new QueryShardContext(indexSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry, percolatorQueryCache);
     }
 
     public void parseFieldMatcher(ParseFieldMatcher parseFieldMatcher) {
@@ -148,6 +153,10 @@ public class QueryShardContext extends QueryRewriteContext {
         return mapperService;
     }
 
+    public PercolatorQueryCache getPercolatorQueryCache() {
+        return percolatorQueryCache;
+    }
+
     public Similarity getSearchSimilarity() {
         return similarityService != null ? similarityService.similarity(mapperService) : null;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
index cd99bec0f74..b1132c42ea9 100644
--- a/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.fieldstats.FieldStatsProvider;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.core.DateFieldMapper;
 import org.joda.time.DateTimeZone;
@@ -253,6 +254,36 @@ public class RangeQueryBuilder extends AbstractQueryBuilder<RangeQueryBuilder> i
         return NAME;
     }
 
+    @Override
+    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        FieldStatsProvider fieldStatsProvider = queryRewriteContext.getFieldStatsProvider();
+        // If the fieldStatsProvider is null we are not on the shard and cannot
+        // rewrite so just return without rewriting
+        if (fieldStatsProvider != null) {
+            DateMathParser dateMathParser = format == null ? null : new DateMathParser(format);
+            FieldStatsProvider.Relation relation = fieldStatsProvider.isFieldWithinQuery(fieldName, from, to, includeLower, includeUpper,
+                    timeZone, dateMathParser);
+            switch (relation) {
+            case DISJOINT:
+                return new MatchNoneQueryBuilder();
+            case WITHIN:
+                if (from != null || to != null) {
+                    RangeQueryBuilder newRangeQuery = new RangeQueryBuilder(fieldName);
+                    newRangeQuery.from(null);
+                    newRangeQuery.to(null);
+                    newRangeQuery.format = format;
+                    newRangeQuery.timeZone = timeZone;
+                    return newRangeQuery;
+                } else {
+                    return this;
+                }
+            case INTERSECTS:
+                break;
+            }
+        }
+        return this;
+    }
+
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
         Query query = null;
diff --git a/core/src/main/java/org/elasticsearch/index/query/support/InnerHitsQueryParserHelper.java b/core/src/main/java/org/elasticsearch/index/query/support/InnerHitsQueryParserHelper.java
index 74a905919f7..0467e459718 100644
--- a/core/src/main/java/org/elasticsearch/index/query/support/InnerHitsQueryParserHelper.java
+++ b/core/src/main/java/org/elasticsearch/index/query/support/InnerHitsQueryParserHelper.java
@@ -27,7 +27,7 @@ import org.elasticsearch.search.fetch.source.FetchSourceParseElement;
 import org.elasticsearch.search.highlight.HighlighterParseElement;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SubSearchContext;
-import org.elasticsearch.search.sort.SortParseElement;
+import org.elasticsearch.search.sort.SortBuilder;
 
 import java.io.IOException;
 
@@ -35,7 +35,6 @@ public class InnerHitsQueryParserHelper {
 
     public static final InnerHitsQueryParserHelper INSTANCE = new InnerHitsQueryParserHelper();
 
-    private static final SortParseElement sortParseElement = new SortParseElement();
     private static final FetchSourceParseElement sourceParseElement = new FetchSourceParseElement();
     private static final HighlighterParseElement highlighterParseElement = new HighlighterParseElement();
     private static final ScriptFieldsParseElement scriptFieldsParseElement = new ScriptFieldsParseElement();
@@ -54,10 +53,10 @@ public class InnerHitsQueryParserHelper {
                     if ("name".equals(fieldName)) {
                         innerHitName = parser.textOrNull();
                     } else {
-                        parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sortParseElement, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement);
+                        parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement);
                     }
                 } else {
-                    parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sortParseElement, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement);
+                    parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement);
                 }
             }
         } catch (Exception e) {
@@ -67,10 +66,10 @@ public class InnerHitsQueryParserHelper {
     }
 
     public static void parseCommonInnerHitOptions(XContentParser parser, XContentParser.Token token, String fieldName, SubSearchContext subSearchContext,
-                                                  SortParseElement sortParseElement, FetchSourceParseElement sourceParseElement, HighlighterParseElement highlighterParseElement,
+                                                  FetchSourceParseElement sourceParseElement, HighlighterParseElement highlighterParseElement,
                                                   ScriptFieldsParseElement scriptFieldsParseElement, FieldDataFieldsParseElement fieldDataFieldsParseElement) throws Exception {
         if ("sort".equals(fieldName)) {
-            sortParseElement.parse(parser, subSearchContext);
+            SortBuilder.parseSort(parser, subSearchContext);
         } else if ("_source".equals(fieldName)) {
             sourceParseElement.parse(parser, subSearchContext);
         } else if (token == XContentParser.Token.START_OBJECT) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/support/NestedInnerQueryParseSupport.java b/core/src/main/java/org/elasticsearch/index/query/support/NestedInnerQueryParseSupport.java
deleted file mode 100644
index 9923728e3bd..00000000000
--- a/core/src/main/java/org/elasticsearch/index/query/support/NestedInnerQueryParseSupport.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.query.support;
-
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.join.BitSetProducer;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.mapper.object.ObjectMapper;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.query.QueryShardException;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-
-/**
- * A helper that helps with parsing inner queries of the nested query.
- * 1) Takes into account that type nested path can appear before or after the inner query
- * 2) Updates the {@link NestedScope} when parsing the inner query.
- */
-public class NestedInnerQueryParseSupport {
-
-    protected final QueryShardContext shardContext;
-    protected final QueryParseContext parseContext;
-
-    private BytesReference source;
-    private Query innerQuery;
-    private Query innerFilter;
-    protected String path;
-
-    private boolean filterParsed = false;
-    private boolean queryParsed = false;
-    protected boolean queryFound = false;
-    protected boolean filterFound = false;
-
-    protected BitSetProducer parentFilter;
-    protected Query childFilter;
-
-    protected ObjectMapper nestedObjectMapper;
-    private ObjectMapper parentObjectMapper;
-
-    public NestedInnerQueryParseSupport(XContentParser parser, SearchContext searchContext) {
-        shardContext = searchContext.getQueryShardContext();
-        parseContext = shardContext.parseContext();
-        shardContext.reset(parser);
-
-    }
-
-    public NestedInnerQueryParseSupport(QueryShardContext context) {
-        this.parseContext = context.parseContext();
-        this.shardContext = context;
-    }
-
-    public void query() throws IOException {
-        if (path != null) {
-            setPathLevel();
-            try {
-                innerQuery = parseContext.parseInnerQueryBuilder().toQuery(this.shardContext);
-            } finally {
-                resetPathLevel();
-            }
-            queryParsed = true;
-        } else {
-            source = XContentFactory.smileBuilder().copyCurrentStructure(parseContext.parser()).bytes();
-        }
-        queryFound = true;
-    }
-
-    public void filter() throws IOException {
-        if (path != null) {
-            setPathLevel();
-            try {
-                innerFilter = QueryBuilder.rewriteQuery(parseContext.parseInnerQueryBuilder(),
-                    this.shardContext).toFilter(this.shardContext);
-            } finally {
-                resetPathLevel();
-            }
-            filterParsed = true;
-        } else {
-            source = XContentFactory.smileBuilder().copyCurrentStructure(parseContext.parser()).bytes();
-        }
-        filterFound = true;
-    }
-
-    public Query getInnerQuery() throws IOException {
-        if (queryParsed) {
-            return innerQuery;
-        } else {
-            if (path == null) {
-                throw new QueryShardException(shardContext, "[nested] requires 'path' field");
-            }
-            if (!queryFound) {
-                throw new QueryShardException(shardContext, "[nested] requires either 'query' or 'filter' field");
-            }
-
-            XContentParser old = parseContext.parser();
-            try {
-                XContentParser innerParser = XContentHelper.createParser(source);
-                parseContext.parser(innerParser);
-                setPathLevel();
-                try {
-                    innerQuery = parseContext.parseInnerQueryBuilder().toQuery(this.shardContext);
-                } finally {
-                    resetPathLevel();
-                }
-                queryParsed = true;
-                return innerQuery;
-            } finally {
-                parseContext.parser(old);
-            }
-        }
-    }
-
-    public Query getInnerFilter() throws IOException {
-        if (filterParsed) {
-            return innerFilter;
-        } else {
-            if (path == null) {
-                throw new QueryShardException(shardContext, "[nested] requires 'path' field");
-            }
-            if (!filterFound) {
-                throw new QueryShardException(shardContext, "[nested] requires either 'query' or 'filter' field");
-            }
-
-            setPathLevel();
-            XContentParser old = parseContext.parser();
-            try {
-                XContentParser innerParser = XContentHelper.createParser(source);
-                parseContext.parser(innerParser);
-                innerFilter = QueryBuilder.rewriteQuery(parseContext.parseInnerQueryBuilder(),
-                    this.shardContext).toFilter(this.shardContext);
-                filterParsed = true;
-                return innerFilter;
-            } finally {
-                resetPathLevel();
-                parseContext.parser(old);
-            }
-        }
-    }
-
-    public void setPath(String path) {
-        this.path = path;
-        nestedObjectMapper = shardContext.getObjectMapper(path);
-        if (nestedObjectMapper == null) {
-            throw new QueryShardException(shardContext, "[nested] failed to find nested object under path [" + path + "]");
-        }
-        if (!nestedObjectMapper.nested().isNested()) {
-            throw new QueryShardException(shardContext, "[nested] nested object under path [" + path + "] is not of nested type");
-        }
-    }
-
-    public String getPath() {
-        return path;
-    }
-
-    public ObjectMapper getNestedObjectMapper() {
-        return nestedObjectMapper;
-    }
-
-    public boolean queryFound() {
-        return queryFound;
-    }
-
-    public boolean filterFound() {
-        return filterFound;
-    }
-
-    public ObjectMapper getParentObjectMapper() {
-        return parentObjectMapper;
-    }
-
-    private void setPathLevel() {
-        ObjectMapper objectMapper = shardContext.nestedScope().getObjectMapper();
-        if (objectMapper == null) {
-            parentFilter = shardContext.bitsetFilter(Queries.newNonNestedFilter());
-        } else {
-            parentFilter = shardContext.bitsetFilter(objectMapper.nestedTypeFilter());
-        }
-        childFilter = nestedObjectMapper.nestedTypeFilter();
-        parentObjectMapper = shardContext.nestedScope().nextLevel(nestedObjectMapper);
-    }
-
-    private void resetPathLevel() {
-        shardContext.nestedScope().previousLevel();
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java b/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java
index 979bfba605f..9cd587704cb 100644
--- a/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java
+++ b/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java
@@ -45,7 +45,6 @@ import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.support.QueryParsers;
 
 import java.io.IOException;
-import java.util.List;
 
 public class MatchQuery {
 
@@ -336,10 +335,10 @@ public class MatchQuery {
                 return prefixQuery;
             } else if (query instanceof MultiPhraseQuery) {
                 MultiPhraseQuery pq = (MultiPhraseQuery)query;
-                List<Term[]> terms = pq.getTermArrays();
+                Term[][] terms = pq.getTermArrays();
                 int[] positions = pq.getPositions();
-                for (int i = 0; i < terms.size(); i++) {
-                    prefixQuery.add(terms.get(i), positions[i]);
+                for (int i = 0; i < terms.length; i++) {
+                    prefixQuery.add(terms[i], positions[i]);
                 }
                 return prefixQuery;
             } else if (query instanceof TermQuery) {
diff --git a/core/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java b/core/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java
index c35a4cdbadb..bcabd7c5e12 100644
--- a/core/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java
+++ b/core/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java
@@ -51,6 +51,10 @@ public class SearchStats implements Streamable, ToXContent {
         private long scrollTimeInMillis;
         private long scrollCurrent;
 
+        private long suggestCount;
+        private long suggestTimeInMillis;
+        private long suggestCurrent;
+
         Stats() {
 
         }
@@ -58,7 +62,8 @@ public class SearchStats implements Streamable, ToXContent {
         public Stats(
                 long queryCount, long queryTimeInMillis, long queryCurrent,
                 long fetchCount, long fetchTimeInMillis, long fetchCurrent,
-                long scrollCount, long scrollTimeInMillis, long scrollCurrent
+                long scrollCount, long scrollTimeInMillis, long scrollCurrent,
+                long suggestCount, long suggestTimeInMillis, long suggestCurrent
         ) {
             this.queryCount = queryCount;
             this.queryTimeInMillis = queryTimeInMillis;
@@ -71,13 +76,19 @@ public class SearchStats implements Streamable, ToXContent {
             this.scrollCount = scrollCount;
             this.scrollTimeInMillis = scrollTimeInMillis;
             this.scrollCurrent = scrollCurrent;
+
+            this.suggestCount = suggestCount;
+            this.suggestTimeInMillis = suggestTimeInMillis;
+            this.suggestCurrent = suggestCurrent;
+
         }
 
         public Stats(Stats stats) {
             this(
                     stats.queryCount, stats.queryTimeInMillis, stats.queryCurrent,
                     stats.fetchCount, stats.fetchTimeInMillis, stats.fetchCurrent,
-                    stats.scrollCount, stats.scrollTimeInMillis, stats.scrollCurrent
+                    stats.scrollCount, stats.scrollTimeInMillis, stats.scrollCurrent,
+                    stats.suggestCount, stats.suggestTimeInMillis, stats.suggestCurrent
             );
         }
 
@@ -93,6 +104,10 @@ public class SearchStats implements Streamable, ToXContent {
             scrollCount += stats.scrollCount;
             scrollTimeInMillis += stats.scrollTimeInMillis;
             scrollCurrent += stats.scrollCurrent;
+
+            suggestCount += stats.suggestCount;
+            suggestTimeInMillis += stats.suggestTimeInMillis;
+            suggestCurrent += stats.suggestCurrent;
         }
 
         public long getQueryCount() {
@@ -143,6 +158,22 @@ public class SearchStats implements Streamable, ToXContent {
             return scrollCurrent;
         }
 
+        public long getSuggestCount() {
+            return suggestCount;
+        }
+
+        public long getSuggestTimeInMillis() {
+            return suggestTimeInMillis;
+        }
+
+        public TimeValue getSuggestTime() {
+            return new TimeValue(suggestTimeInMillis);
+        }
+
+        public long getSuggestCurrent() {
+            return suggestCurrent;
+        }
+
         public static Stats readStats(StreamInput in) throws IOException {
             Stats stats = new Stats();
             stats.readFrom(in);
@@ -162,6 +193,10 @@ public class SearchStats implements Streamable, ToXContent {
             scrollCount = in.readVLong();
             scrollTimeInMillis = in.readVLong();
             scrollCurrent = in.readVLong();
+
+            suggestCount = in.readVLong();
+            suggestTimeInMillis = in.readVLong();
+            suggestCurrent = in.readVLong();
         }
 
         @Override
@@ -177,6 +212,10 @@ public class SearchStats implements Streamable, ToXContent {
             out.writeVLong(scrollCount);
             out.writeVLong(scrollTimeInMillis);
             out.writeVLong(scrollCurrent);
+
+            out.writeVLong(suggestCount);
+            out.writeVLong(suggestTimeInMillis);
+            out.writeVLong(suggestCurrent);
         }
 
         @Override
@@ -193,6 +232,10 @@ public class SearchStats implements Streamable, ToXContent {
             builder.timeValueField(Fields.SCROLL_TIME_IN_MILLIS, Fields.SCROLL_TIME, scrollTimeInMillis);
             builder.field(Fields.SCROLL_CURRENT, scrollCurrent);
 
+            builder.field(Fields.SUGGEST_TOTAL, suggestCount);
+            builder.timeValueField(Fields.SUGGEST_TIME_IN_MILLIS, Fields.SUGGEST_TIME, suggestTimeInMillis);
+            builder.field(Fields.SUGGEST_CURRENT, suggestCurrent);
+
             return builder;
         }
     }
@@ -292,6 +335,10 @@ public class SearchStats implements Streamable, ToXContent {
         static final XContentBuilderString SCROLL_TIME = new XContentBuilderString("scroll_time");
         static final XContentBuilderString SCROLL_TIME_IN_MILLIS = new XContentBuilderString("scroll_time_in_millis");
         static final XContentBuilderString SCROLL_CURRENT = new XContentBuilderString("scroll_current");
+        static final XContentBuilderString SUGGEST_TOTAL = new XContentBuilderString("suggest_total");
+        static final XContentBuilderString SUGGEST_TIME = new XContentBuilderString("suggest_time");
+        static final XContentBuilderString SUGGEST_TIME_IN_MILLIS = new XContentBuilderString("suggest_time_in_millis");
+        static final XContentBuilderString SUGGEST_CURRENT = new XContentBuilderString("suggest_current");
     }
 
     public static SearchStats readSearchStats(StreamInput in) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java b/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
index 1a155d17964..748bb01bd54 100644
--- a/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
+++ b/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
@@ -23,13 +23,13 @@ import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.metrics.MeanMetric;
 import org.elasticsearch.common.regex.Regex;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.SearchSlowLog;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
 
 import static java.util.Collections.emptyMap;
 
@@ -72,64 +72,51 @@ public final class ShardSearchStats {
     }
 
     public void onPreQueryPhase(SearchContext searchContext) {
-        totalStats.queryCurrent.inc();
-        if (searchContext.groupStats() != null) {
-            for (int i = 0; i < searchContext.groupStats().size(); i++) {
-                groupStats(searchContext.groupStats().get(i)).queryCurrent.inc();
+        computeStats(searchContext, statsHolder -> {
+            if (searchContext.hasOnlySuggest()) {
+                statsHolder.suggestCurrent.inc();
+            } else {
+                statsHolder.queryCurrent.inc();
             }
-        }
+        });
     }
 
     public void onFailedQueryPhase(SearchContext searchContext) {
-        totalStats.queryCurrent.dec();
-        if (searchContext.groupStats() != null) {
-            for (int i = 0; i < searchContext.groupStats().size(); i++) {
-                groupStats(searchContext.groupStats().get(i)).queryCurrent.dec();
+        computeStats(searchContext, statsHolder -> {
+            if (searchContext.hasOnlySuggest()) {
+                statsHolder.suggestCurrent.dec();
+            } else {
+                statsHolder.queryCurrent.dec();
             }
-        }
+        });
     }
 
     public void onQueryPhase(SearchContext searchContext, long tookInNanos) {
-        totalStats.queryMetric.inc(tookInNanos);
-        totalStats.queryCurrent.dec();
-        if (searchContext.groupStats() != null) {
-            for (int i = 0; i < searchContext.groupStats().size(); i++) {
-                StatsHolder statsHolder = groupStats(searchContext.groupStats().get(i));
+        computeStats(searchContext, statsHolder -> {
+            if (searchContext.hasOnlySuggest()) {
+                statsHolder.suggestMetric.inc(tookInNanos);
+                statsHolder.suggestCurrent.dec();
+            } else {
                 statsHolder.queryMetric.inc(tookInNanos);
                 statsHolder.queryCurrent.dec();
             }
-        }
+        });
         slowLogSearchService.onQueryPhase(searchContext, tookInNanos);
     }
 
     public void onPreFetchPhase(SearchContext searchContext) {
-        totalStats.fetchCurrent.inc();
-        if (searchContext.groupStats() != null) {
-            for (int i = 0; i < searchContext.groupStats().size(); i++) {
-                groupStats(searchContext.groupStats().get(i)).fetchCurrent.inc();
-            }
-        }
+        computeStats(searchContext, statsHolder -> statsHolder.fetchCurrent.inc());
     }
 
     public void onFailedFetchPhase(SearchContext searchContext) {
-        totalStats.fetchCurrent.dec();
-        if (searchContext.groupStats() != null) {
-            for (int i = 0; i < searchContext.groupStats().size(); i++) {
-                groupStats(searchContext.groupStats().get(i)).fetchCurrent.dec();
-            }
-        }
+        computeStats(searchContext, statsHolder -> statsHolder.fetchCurrent.dec());
     }
 
     public void onFetchPhase(SearchContext searchContext, long tookInNanos) {
-        totalStats.fetchMetric.inc(tookInNanos);
-        totalStats.fetchCurrent.dec();
-        if (searchContext.groupStats() != null) {
-            for (int i = 0; i < searchContext.groupStats().size(); i++) {
-                StatsHolder statsHolder = groupStats(searchContext.groupStats().get(i));
-                statsHolder.fetchMetric.inc(tookInNanos);
-                statsHolder.fetchCurrent.dec();
-            }
-        }
+        computeStats(searchContext, statsHolder -> {
+            statsHolder.fetchMetric.inc(tookInNanos);
+            statsHolder.fetchCurrent.dec();
+        });
         slowLogSearchService.onFetchPhase(searchContext, tookInNanos);
     }
 
@@ -149,6 +136,15 @@ public final class ShardSearchStats {
         }
     }
 
+    private void computeStats(SearchContext searchContext, Consumer<StatsHolder> consumer) {
+        consumer.accept(totalStats);
+        if (searchContext.groupStats() != null) {
+            for (int i = 0; i < searchContext.groupStats().size(); i++) {
+                consumer.accept(groupStats(searchContext.groupStats().get(i)));
+            }
+        }
+    }
+
     private StatsHolder groupStats(String group) {
         StatsHolder stats = groupsStats.get(group);
         if (stats == null) {
@@ -184,26 +180,30 @@ public final class ShardSearchStats {
         public final MeanMetric queryMetric = new MeanMetric();
         public final MeanMetric fetchMetric = new MeanMetric();
         public final MeanMetric scrollMetric = new MeanMetric();
+        public final MeanMetric suggestMetric = new MeanMetric();
         public final CounterMetric queryCurrent = new CounterMetric();
         public final CounterMetric fetchCurrent = new CounterMetric();
         public final CounterMetric scrollCurrent = new CounterMetric();
+        public final CounterMetric suggestCurrent = new CounterMetric();
 
         public SearchStats.Stats stats() {
             return new SearchStats.Stats(
                     queryMetric.count(), TimeUnit.NANOSECONDS.toMillis(queryMetric.sum()), queryCurrent.count(),
                     fetchMetric.count(), TimeUnit.NANOSECONDS.toMillis(fetchMetric.sum()), fetchCurrent.count(),
-                    scrollMetric.count(), TimeUnit.NANOSECONDS.toMillis(scrollMetric.sum()), scrollCurrent.count()
+                    scrollMetric.count(), TimeUnit.NANOSECONDS.toMillis(scrollMetric.sum()), scrollCurrent.count(),
+                    suggestMetric.count(), TimeUnit.NANOSECONDS.toMillis(suggestMetric.sum()), suggestCurrent.count()
             );
         }
 
         public long totalCurrent() {
-            return queryCurrent.count() + fetchCurrent.count() + scrollCurrent.count();
+            return queryCurrent.count() + fetchCurrent.count() + scrollCurrent.count() + suggestCurrent.count();
         }
 
         public void clear() {
             queryMetric.clear();
             fetchMetric.clear();
             scrollMetric.clear();
+            suggestMetric.clear();
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ElasticsearchMergePolicy.java b/core/src/main/java/org/elasticsearch/index/shard/ElasticsearchMergePolicy.java
index 524266420fb..adae6caf452 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ElasticsearchMergePolicy.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ElasticsearchMergePolicy.java
@@ -49,7 +49,7 @@ import java.util.Map;
  * be stored as payloads to numeric doc values.
  */
 public final class ElasticsearchMergePolicy extends MergePolicy {
-    
+
     private static ESLogger logger = Loggers.getLogger(ElasticsearchMergePolicy.class);
 
     private final MergePolicy delegate;
@@ -69,7 +69,7 @@ public final class ElasticsearchMergePolicy extends MergePolicy {
 
     /** Return an "upgraded" view of the reader. */
     static CodecReader filter(CodecReader reader) throws IOException {
-        // TODO: remove 0.90.x/1.x freqs/prox/payloads from _uid? 
+        // TODO: remove 0.90.x/1.x freqs/prox/payloads from _uid?
         // the previous code never did this, so some indexes carry around trash.
         return reader;
     }
@@ -155,7 +155,7 @@ public final class ElasticsearchMergePolicy extends MergePolicy {
 
                     // TODO: Use IndexUpgradeMergePolicy instead.  We should be comparing codecs,
                     // for now we just assume every minor upgrade has a new format.
-                    logger.debug("Adding segment " + info.info.name + " to be upgraded");
+                    logger.debug("Adding segment {} to be upgraded", info.info.name);
                     spec.add(new OneMerge(Collections.singletonList(info)));
                 }
 
@@ -163,14 +163,14 @@ public final class ElasticsearchMergePolicy extends MergePolicy {
 
                 if (spec.merges.size() == MAX_CONCURRENT_UPGRADE_MERGES) {
                     // hit our max upgrades, so return the spec.  we will get a cascaded call to continue.
-                    logger.debug("Returning " + spec.merges.size() + " merges for upgrade");
+                    logger.debug("Returning {} merges for upgrade", spec.merges.size());
                     return spec;
                 }
             }
 
             // We must have less than our max upgrade merges, so the next return will be our last in upgrading mode.
             if (spec.merges.isEmpty() == false) {
-                logger.debug("Returning " + spec.merges.size() + " merges for end of upgrade");
+                logger.debug("Returning {} merges for end of upgrade", spec.merges.size());
                 return spec;
             }
 
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IllegalIndexShardStateException.java b/core/src/main/java/org/elasticsearch/index/shard/IllegalIndexShardStateException.java
index 626e72acf41..e632c0669f6 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IllegalIndexShardStateException.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IllegalIndexShardStateException.java
@@ -33,12 +33,12 @@ public class IllegalIndexShardStateException extends ElasticsearchException {
 
     private final IndexShardState currentState;
 
-    public IllegalIndexShardStateException(ShardId shardId, IndexShardState currentState, String msg) {
-        this(shardId, currentState, msg, null);
+    public IllegalIndexShardStateException(ShardId shardId, IndexShardState currentState, String msg, Object... args) {
+        this(shardId, currentState, msg, null, args);
     }
 
-    public IllegalIndexShardStateException(ShardId shardId, IndexShardState currentState, String msg, Throwable ex) {
-        super("CurrentState[" + currentState + "] " + msg, ex);
+    public IllegalIndexShardStateException(ShardId shardId, IndexShardState currentState, String msg, Throwable ex, Object... args) {
+        super("CurrentState[" + currentState + "] " + msg, ex, args);
         setShard(shardId);
         this.currentState = currentState;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 168f381d2d7..5e9d078a078 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -41,19 +41,18 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.metrics.MeanMetric;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.common.util.concurrent.SuspendableRefContainer;
-import org.elasticsearch.gateway.MetaDataStateFormat;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.SearchSlowLog;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.cache.IndexCache;
@@ -83,9 +82,7 @@ import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.merge.MergeStats;
-import org.elasticsearch.index.percolator.PercolateStats;
-import org.elasticsearch.index.percolator.PercolatorQueriesRegistry;
-import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.percolator.PercolatorFieldMapper;
 import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.refresh.RefreshStats;
 import org.elasticsearch.index.search.stats.SearchStats;
@@ -96,8 +93,6 @@ import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.store.Store.MetadataSnapshot;
 import org.elasticsearch.index.store.StoreFileMetaData;
 import org.elasticsearch.index.store.StoreStats;
-import org.elasticsearch.index.suggest.stats.ShardSuggestMetric;
-import org.elasticsearch.index.suggest.stats.SuggestStats;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogConfig;
 import org.elasticsearch.index.translog.TranslogStats;
@@ -106,7 +101,6 @@ import org.elasticsearch.index.warmer.WarmerStats;
 import org.elasticsearch.indices.IndexingMemoryController;
 import org.elasticsearch.indices.recovery.RecoveryFailedException;
 import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.percolator.PercolatorService;
 import org.elasticsearch.search.suggest.completion.CompletionFieldStats;
 import org.elasticsearch.search.suggest.completion.CompletionStats;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -122,7 +116,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
@@ -140,9 +133,7 @@ public class IndexShard extends AbstractIndexShardComponent {
     private final ShardIndexWarmerService shardWarmerService;
     private final ShardRequestCache shardQueryCache;
     private final ShardFieldData shardFieldData;
-    private final PercolatorQueriesRegistry percolatorQueriesRegistry;
     private final IndexFieldDataService indexFieldDataService;
-    private final ShardSuggestMetric shardSuggestMetric = new ShardSuggestMetric();
     private final ShardBitsetFilterCache shardBitsetFilterCache;
     private final Object mutex = new Object();
     private final String checkIndexOnStartup;
@@ -154,14 +145,16 @@ public class IndexShard extends AbstractIndexShardComponent {
     private final TranslogConfig translogConfig;
     private final IndexEventListener indexEventListener;
 
-    /** How many bytes we are currently moving to disk, via either IndexWriter.flush or refresh.  IndexingMemoryController polls this
-     *  across all shards to decide if throttling is necessary because moving bytes to disk is falling behind vs incoming documents
-     *  being indexed/deleted. */
+    /**
+     * How many bytes we are currently moving to disk, via either IndexWriter.flush or refresh.  IndexingMemoryController polls this
+     * across all shards to decide if throttling is necessary because moving bytes to disk is falling behind vs incoming documents
+     * being indexed/deleted.
+     */
     private final AtomicLong writingBytes = new AtomicLong();
 
-    private volatile ScheduledFuture<?> refreshScheduledFuture;
     protected volatile ShardRouting shardRouting;
     protected volatile IndexShardState state;
+    protected volatile long primaryTerm;
     protected final AtomicReference<Engine> currentEngineReference = new AtomicReference<>();
     protected final EngineFactory engineFactory;
 
@@ -201,7 +194,8 @@ public class IndexShard extends AbstractIndexShardComponent {
     public IndexShard(ShardId shardId, IndexSettings indexSettings, ShardPath path, Store store, IndexCache indexCache,
                       MapperService mapperService, SimilarityService similarityService, IndexFieldDataService indexFieldDataService,
                       @Nullable EngineFactory engineFactory,
-                      IndexEventListener indexEventListener, IndexSearcherWrapper indexSearcherWrapper, NodeServicesProvider provider, SearchSlowLog slowLog, Engine.Warmer warmer, IndexingOperationListener... listeners) {
+                      IndexEventListener indexEventListener, IndexSearcherWrapper indexSearcherWrapper, ThreadPool threadPool, BigArrays bigArrays,
+                      SearchSlowLog slowLog, Engine.Warmer warmer, IndexingOperationListener... listeners) {
         super(shardId, indexSettings);
         final Settings settings = indexSettings.getSettings();
         this.codecService = new CodecService(mapperService, logger);
@@ -212,7 +206,7 @@ public class IndexShard extends AbstractIndexShardComponent {
         this.engineFactory = engineFactory == null ? new InternalEngineFactory() : engineFactory;
         this.store = store;
         this.indexEventListener = indexEventListener;
-        this.threadPool = provider.getThreadPool();
+        this.threadPool = threadPool;
         this.mapperService = mapperService;
         this.indexCache = indexCache;
         this.internalIndexingStats = new InternalIndexingStats();
@@ -233,7 +227,7 @@ public class IndexShard extends AbstractIndexShardComponent {
 
         this.checkIndexOnStartup = indexSettings.getValue(IndexSettings.INDEX_CHECK_ON_STARTUP);
         this.translogConfig = new TranslogConfig(shardId, shardPath().resolveTranslog(), indexSettings,
-            provider.getBigArrays());
+            bigArrays);
         final QueryCachingPolicy cachingPolicy;
         // the query cache is a node-level thing, however we want the most popular filters
         // to be computed on a per-shard basis
@@ -246,15 +240,16 @@ public class IndexShard extends AbstractIndexShardComponent {
         this.engineConfig = newEngineConfig(translogConfig, cachingPolicy);
         this.suspendableRefContainer = new SuspendableRefContainer();
         this.searcherWrapper = indexSearcherWrapper;
-        QueryShardContext queryShardContext = new QueryShardContext(indexSettings, indexCache.bitsetFilterCache(), indexFieldDataService, mapperService, similarityService, provider.getScriptService(), provider.getIndicesQueriesRegistry());
-        this.percolatorQueriesRegistry = new PercolatorQueriesRegistry(shardId, indexSettings, queryShardContext);
+        this.primaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardId.id());
     }
 
     public Store store() {
         return this.store;
     }
 
-    /** returns true if this shard supports indexing (i.e., write) operations. */
+    /**
+     * returns true if this shard supports indexing (i.e., write) operations.
+     */
     public boolean canIndex() {
         return true;
     }
@@ -263,10 +258,6 @@ public class IndexShard extends AbstractIndexShardComponent {
         return this.getService;
     }
 
-    public ShardSuggestMetric getSuggestMetric() {
-        return shardSuggestMetric;
-    }
-
     public ShardBitsetFilterCache shardBitsetFilterCache() {
         return shardBitsetFilterCache;
     }
@@ -295,6 +286,30 @@ public class IndexShard extends AbstractIndexShardComponent {
         return this.shardFieldData;
     }
 
+
+    /**
+     * Returns the primary term the index shard is on. See {@link org.elasticsearch.cluster.metadata.IndexMetaData#primaryTerm(int)}
+     */
+    public long getPrimaryTerm() {
+        return this.primaryTerm;
+    }
+
+    /**
+     * notifies the shard of an increase in the primary term
+     */
+    public void updatePrimaryTerm(final long newTerm) {
+        synchronized (mutex) {
+            if (newTerm != primaryTerm) {
+                assert shardRouting.primary() == false : "a primary shard should never update it's term. shard: " + shardRouting
+                    + " current term [" + primaryTerm + "] new term [" + newTerm + "]";
+                assert newTerm > primaryTerm : "primary terms can only go up. current [" + primaryTerm + "], new [" + newTerm + "]";
+                primaryTerm = newTerm;
+            }
+        }
+
+
+    }
+
     /**
      * Returns the latest cluster routing entry received with this shard. Might be null if the
      * shard was just created.
@@ -313,12 +328,12 @@ public class IndexShard extends AbstractIndexShardComponent {
      * unless explicitly disabled.
      *
      * @throws IndexShardRelocatedException if shard is marked as relocated and relocation aborted
-     * @throws IOException if shard state could not be persisted
+     * @throws IOException                  if shard state could not be persisted
      */
     public void updateRoutingEntry(final ShardRouting newRouting, final boolean persistState) throws IOException {
         final ShardRouting currentRouting = this.shardRouting;
         if (!newRouting.shardId().equals(shardId())) {
-            throw new IllegalArgumentException("Trying to set a routing entry with shardId [" + newRouting.shardId() + "] on a shard with shardId [" + shardId() + "]");
+            throw new IllegalArgumentException("Trying to set a routing entry with shardId " + newRouting.shardId() + " on a shard with shardId " + shardId() + "");
         }
         if ((currentRouting == null || newRouting.isSameAllocation(currentRouting)) == false) {
             throw new IllegalArgumentException("Trying to set a routing entry with a different allocation. Current " + currentRouting + ", new " + newRouting);
@@ -435,9 +450,7 @@ public class IndexShard extends AbstractIndexShardComponent {
 
     public Engine.Index prepareIndexOnPrimary(SourceToParse source, long version, VersionType versionType) {
         try {
-            if (shardRouting.primary() == false) {
-                throw new IllegalIndexShardStateException(shardId, state, "shard is not a primary");
-            }
+            verifyPrimary();
             return prepareIndex(docMapper(source.type()), source, version, versionType, Engine.Operation.Origin.PRIMARY);
         } catch (Throwable t) {
             verifyNotClosed(t);
@@ -447,6 +460,7 @@ public class IndexShard extends AbstractIndexShardComponent {
 
     public Engine.Index prepareIndexOnReplica(SourceToParse source, long version, VersionType versionType) {
         try {
+            verifyReplicationTarget();
             return prepareIndex(docMapper(source.type()), source, version, versionType, Engine.Operation.Origin.REPLICA);
         } catch (Throwable t) {
             verifyNotClosed(t);
@@ -476,12 +490,8 @@ public class IndexShard extends AbstractIndexShardComponent {
             if (logger.isTraceEnabled()) {
                 logger.trace("index [{}][{}]{}", index.type(), index.id(), index.docs());
             }
-            final boolean isPercolatorQuery = percolatorQueriesRegistry.isPercolatorQuery(index);
             Engine engine = getEngine();
             created = engine.index(index);
-            if (isPercolatorQuery) {
-                percolatorQueriesRegistry.updatePercolateQuery(engine, index.id());
-            }
             index.endTime(System.nanoTime());
         } catch (Throwable ex) {
             indexingOperationListeners.postIndex(index, ex);
@@ -494,9 +504,7 @@ public class IndexShard extends AbstractIndexShardComponent {
     }
 
     public Engine.Delete prepareDeleteOnPrimary(String type, String id, long version, VersionType versionType) {
-        if (shardRouting.primary() == false) {
-            throw new IllegalIndexShardStateException(shardId, state, "shard is not a primary");
-        }
+        verifyPrimary();
         final DocumentMapper documentMapper = docMapper(type).getDocumentMapper();
         return prepareDelete(type, id, documentMapper.uidMapper().term(Uid.createUid(type, id)), version, versionType, Engine.Operation.Origin.PRIMARY);
     }
@@ -519,12 +527,8 @@ public class IndexShard extends AbstractIndexShardComponent {
             if (logger.isTraceEnabled()) {
                 logger.trace("delete [{}]", delete.uid().text());
             }
-            final boolean isPercolatorQuery = percolatorQueriesRegistry.isPercolatorQuery(delete);
             Engine engine = getEngine();
             engine.delete(delete);
-            if (isPercolatorQuery) {
-                percolatorQueriesRegistry.updatePercolateQuery(engine, delete.id());
-            }
             delete.endTime(System.nanoTime());
         } catch (Throwable ex) {
             indexingOperationListeners.postDelete(delete, ex);
@@ -539,7 +543,9 @@ public class IndexShard extends AbstractIndexShardComponent {
         return getEngine().get(get, this::acquireSearcher);
     }
 
-    /** Writes all indexing changes to disk and opens a new searcher reflecting all changes.  This can throw {@link EngineClosedException}. */
+    /**
+     * Writes all indexing changes to disk and opens a new searcher reflecting all changes.  This can throw {@link EngineClosedException}.
+     */
     public void refresh(String source) {
         verifyNotClosed();
         if (canIndex()) {
@@ -562,7 +568,9 @@ public class IndexShard extends AbstractIndexShardComponent {
         }
     }
 
-    /** Returns how many bytes we are currently moving from heap to disk */
+    /**
+     * Returns how many bytes we are currently moving from heap to disk
+     */
     public long getWritingBytes() {
         return writingBytes.get();
     }
@@ -644,18 +652,10 @@ public class IndexShard extends AbstractIndexShardComponent {
         return shardFieldData.stats(fields);
     }
 
-    public PercolatorQueriesRegistry percolateRegistry() {
-        return percolatorQueriesRegistry;
-    }
-
     public TranslogStats translogStats() {
         return getEngine().getTranslog().stats();
     }
 
-    public SuggestStats suggestStats() {
-        return shardSuggestMetric.stats();
-    }
-
     public CompletionStats completionStats(String... fields) {
         CompletionStats completionStats = new CompletionStats();
         try (final Engine.Searcher currentSearcher = acquireSearcher("completion_stats")) {
@@ -712,7 +712,7 @@ public class IndexShard extends AbstractIndexShardComponent {
             false, true, upgrade.upgradeOnlyAncientSegments());
         org.apache.lucene.util.Version version = minimumCompatibleVersion();
         if (logger.isTraceEnabled()) {
-            logger.trace("upgraded segment {} from version {} to version {}", previousVersion, version);
+            logger.trace("upgraded segments for {} from version {} to version {}", shardId, previousVersion, version);
         }
 
         return version;
@@ -790,10 +790,6 @@ public class IndexShard extends AbstractIndexShardComponent {
     public void close(String reason, boolean flushEngine) throws IOException {
         synchronized (mutex) {
             try {
-                if (state != IndexShardState.CLOSED) {
-                    FutureUtils.cancel(refreshScheduledFuture);
-                    refreshScheduledFuture = null;
-                }
                 changeState(IndexShardState.CLOSED, reason);
             } finally {
                 final Engine engine = this.currentEngineReference.getAndSet(null);
@@ -802,18 +798,15 @@ public class IndexShard extends AbstractIndexShardComponent {
                         engine.flushAndClose();
                     }
                 } finally { // playing safe here and close the engine even if the above succeeds - close can be called multiple times
-                    IOUtils.close(engine, percolatorQueriesRegistry);
+                    IOUtils.close(engine);
                 }
             }
         }
     }
 
     public IndexShard postRecovery(String reason) throws IndexShardStartedException, IndexShardRelocatedException, IndexShardClosedException {
-        if (mapperService.hasMapping(PercolatorService.TYPE_NAME)) {
+        if (mapperService.hasMapping(PercolatorFieldMapper.TYPE_NAME)) {
             refresh("percolator_load_queries");
-            try (Engine.Searcher searcher = getEngine().acquireSearcher("percolator_load_queries")) {
-                this.percolatorQueriesRegistry.loadQueries(searcher.reader());
-            }
         }
         synchronized (mutex) {
             if (state == IndexShardState.CLOSED) {
@@ -985,6 +978,22 @@ public class IndexShard extends AbstractIndexShardComponent {
         }
     }
 
+    private void verifyPrimary() {
+        if (shardRouting.primary() == false) {
+            // must use exception that is not ignored by replication logic. See TransportActions.isShardNotAvailableException
+            throw new IllegalStateException("shard is not a primary " + shardRouting);
+        }
+    }
+
+    private void verifyReplicationTarget() {
+        final IndexShardState state = state();
+        if (shardRouting.primary() && shardRouting.active() && state != IndexShardState.RELOCATED) {
+            // must use exception that is not ignored by replication logic. See TransportActions.isShardNotAvailableException
+            throw new IllegalStateException("active primary shard cannot be a replication target before " +
+                " relocation hand off " + shardRouting + ", state is [" + state + "]");
+        }
+    }
+
     protected final void verifyStartedOrRecovering() throws IllegalIndexShardStateException {
         IndexShardState state = this.state; // one time volatile read
         if (state != IndexShardState.STARTED && state != IndexShardState.RECOVERING && state != IndexShardState.POST_RECOVERY) {
@@ -1014,7 +1023,9 @@ public class IndexShard extends AbstractIndexShardComponent {
         }
     }
 
-    /** Returns number of heap bytes used by the indexing buffer for this shard, or 0 if the shard is closed */
+    /**
+     * Returns number of heap bytes used by the indexing buffer for this shard, or 0 if the shard is closed
+     */
     public long getIndexBufferRAMBytesUsed() {
         Engine engine = getEngineOrNull();
         if (engine == null) {
@@ -1031,8 +1042,10 @@ public class IndexShard extends AbstractIndexShardComponent {
         this.shardEventListener.delegates.add(onShardFailure);
     }
 
-    /** Called by {@link IndexingMemoryController} to check whether more than {@code inactiveTimeNS} has passed since the last
-     *  indexing operation, and notify listeners that we are now inactive so e.g. sync'd flush can happen. */
+    /**
+     * Called by {@link IndexingMemoryController} to check whether more than {@code inactiveTimeNS} has passed since the last
+     * indexing operation, and notify listeners that we are now inactive so e.g. sync'd flush can happen.
+     */
     public void checkIdle(long inactiveTimeNS) {
         Engine engineOrNull = getEngineOrNull();
         if (engineOrNull != null && System.nanoTime() - engineOrNull.getLastWriteNanos() >= inactiveTimeNS) {
@@ -1110,10 +1123,6 @@ public class IndexShard extends AbstractIndexShardComponent {
         return getEngine().getTranslog();
     }
 
-    public PercolateStats percolateStats() {
-        return percolatorQueriesRegistry.stats();
-    }
-
     public IndexEventListener getIndexEventListener() {
         return indexEventListener;
     }
@@ -1181,11 +1190,12 @@ public class IndexShard extends AbstractIndexShardComponent {
             }
         } catch (Exception e) {
             handleRefreshException(e);
-        };
+        }
     }
 
     /**
      * Should be called for each no-op update operation to increment relevant statistics.
+     *
      * @param type the doc type of the update
      */
     public void noopUpdate(String type) {
@@ -1385,14 +1395,22 @@ public class IndexShard extends AbstractIndexShardComponent {
 
     public Releasable acquirePrimaryOperationLock() {
         verifyNotClosed();
-        if (shardRouting.primary() == false) {
-            throw new IllegalIndexShardStateException(shardId, state, "shard is not a primary");
-        }
+        verifyPrimary();
         return suspendableRefContainer.acquireUninterruptibly();
     }
 
-    public Releasable acquireReplicaOperationLock() {
+    /**
+     * acquires operation log. If the given primary term is lower then the one in {@link #shardRouting}
+     * an {@link IllegalArgumentException} is thrown.
+     */
+    public Releasable acquireReplicaOperationLock(long opPrimaryTerm) {
         verifyNotClosed();
+        verifyReplicationTarget();
+        if (primaryTerm > opPrimaryTerm) {
+            // must use exception that is not ignored by replication logic. See TransportActions.isShardNotAvailableException
+            throw new IllegalArgumentException(LoggerMessageFormat.format("{} operation term [{}] is too old (current [{}])",
+                shardId, opPrimaryTerm, primaryTerm));
+        }
         return suspendableRefContainer.acquireUninterruptibly();
     }
 
@@ -1496,7 +1514,7 @@ public class IndexShard extends AbstractIndexShardComponent {
      * Returns <code>true</code> iff one or more changes to the engine are not visible to via the current searcher.
      * Otherwise <code>false</code>.
      *
-     * @throws EngineClosedException if the engine is already closed
+     * @throws EngineClosedException  if the engine is already closed
      * @throws AlreadyClosedException if the internal indexwriter in the engine is already closed
      */
     public boolean isRefreshNeeded() {
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
index 5518d1b1273..774052b3a5f 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
@@ -20,8 +20,8 @@ package org.elasticsearch.index.shard;
 
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.cache.IndexCache;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineConfig;
@@ -33,6 +33,7 @@ import org.elasticsearch.index.SearchSlowLog;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.TranslogStats;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 
@@ -44,9 +45,13 @@ import java.io.IOException;
  */
 public final class ShadowIndexShard extends IndexShard {
 
-    public ShadowIndexShard(ShardId shardId, IndexSettings indexSettings, ShardPath path, Store store, IndexCache indexCache, MapperService mapperService, SimilarityService similarityService, IndexFieldDataService indexFieldDataService, @Nullable EngineFactory engineFactory,
-                            IndexEventListener indexEventListener, IndexSearcherWrapper wrapper, NodeServicesProvider provider, SearchSlowLog searchSlowLog, Engine.Warmer engineWarmer) throws IOException {
-        super(shardId, indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldDataService, engineFactory, indexEventListener, wrapper, provider, searchSlowLog, engineWarmer);
+    public ShadowIndexShard(ShardId shardId, IndexSettings indexSettings, ShardPath path, Store store, IndexCache indexCache,
+                            MapperService mapperService, SimilarityService similarityService, IndexFieldDataService indexFieldDataService,
+                            @Nullable EngineFactory engineFactory, IndexEventListener indexEventListener, IndexSearcherWrapper wrapper,
+                            ThreadPool threadPool, BigArrays bigArrays, SearchSlowLog searchSlowLog, Engine.Warmer engineWarmer)
+        throws IOException {
+        super(shardId, indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldDataService, engineFactory,
+            indexEventListener, wrapper, threadPool, bigArrays, searchSlowLog, engineWarmer);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardId.java b/core/src/main/java/org/elasticsearch/index/shard/ShardId.java
index 3dea5501c62..a9bc63ae44f 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ShardId.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ShardId.java
@@ -98,7 +98,7 @@ public class ShardId implements Streamable, Comparable<ShardId> {
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
-        index = Index.readIndex(in);
+        index = new Index(in);
         shardId = in.readVInt();
         hashCode = computeHashCode();
     }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java b/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java
index 3d6fbf08102..be0d51bd2b6 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java
@@ -29,7 +29,6 @@ import java.io.IOException;
 import java.nio.file.FileStore;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.HashMap;
 import java.util.Map;
 
 public final class ShardPath {
@@ -37,22 +36,20 @@ public final class ShardPath {
     public static final String TRANSLOG_FOLDER_NAME = "translog";
 
     private final Path path;
-    private final String indexUUID;
     private final ShardId shardId;
     private final Path shardStatePath;
     private final boolean isCustomDataPath;
 
-    public ShardPath(boolean isCustomDataPath, Path dataPath, Path shardStatePath, String indexUUID, ShardId shardId) {
+    public ShardPath(boolean isCustomDataPath, Path dataPath, Path shardStatePath, ShardId shardId) {
         assert dataPath.getFileName().toString().equals(Integer.toString(shardId.id())) : "dataPath must end with the shard ID but didn't: " + dataPath.toString();
         assert shardStatePath.getFileName().toString().equals(Integer.toString(shardId.id())) : "shardStatePath must end with the shard ID but didn't: " + dataPath.toString();
-        assert dataPath.getParent().getFileName().toString().equals(shardId.getIndexName()) : "dataPath must end with index/shardID but didn't: " + dataPath.toString();
-        assert shardStatePath.getParent().getFileName().toString().equals(shardId.getIndexName()) : "shardStatePath must end with index/shardID but didn't: " + dataPath.toString();
+        assert dataPath.getParent().getFileName().toString().equals(shardId.getIndex().getUUID()) : "dataPath must end with index path id but didn't: " + dataPath.toString();
+        assert shardStatePath.getParent().getFileName().toString().equals(shardId.getIndex().getUUID()) : "shardStatePath must end with index path id but didn't: " + dataPath.toString();
         if (isCustomDataPath && dataPath.equals(shardStatePath)) {
             throw new IllegalArgumentException("shard state path must be different to the data path when using custom data paths");
         }
         this.isCustomDataPath = isCustomDataPath;
         this.path = dataPath;
-        this.indexUUID = indexUUID;
         this.shardId = shardId;
         this.shardStatePath = shardStatePath;
     }
@@ -73,10 +70,6 @@ public final class ShardPath {
         return Files.exists(path);
     }
 
-    public String getIndexUUID() {
-        return indexUUID;
-    }
-
     public ShardId getShardId() {
         return shardId;
     }
@@ -144,7 +137,7 @@ public final class ShardPath {
                 dataPath = statePath;
             }
             logger.debug("{} loaded data path [{}], state path [{}]", shardId, dataPath, statePath);
-            return new ShardPath(indexSettings.hasCustomDataPath(), dataPath, statePath, indexUUID, shardId);
+            return new ShardPath(indexSettings.hasCustomDataPath(), dataPath, statePath, shardId);
         }
     }
 
@@ -168,34 +161,6 @@ public final class ShardPath {
         }
     }
 
-    /** Maps each path.data path to a "guess" of how many bytes the shards allocated to that path might additionally use over their
-     *  lifetime; we do this so a bunch of newly allocated shards won't just all go the path with the most free space at this moment. */
-    private static Map<Path,Long> getEstimatedReservedBytes(NodeEnvironment env, long avgShardSizeInBytes, Iterable<IndexShard> shards) throws IOException {
-        long totFreeSpace = 0;
-        for (NodeEnvironment.NodePath nodePath : env.nodePaths()) {
-            totFreeSpace += nodePath.fileStore.getUsableSpace();
-        }
-
-        // Very rough heuristic of how much disk space we expect the shard will use over its lifetime, the max of current average
-        // shard size across the cluster and 5% of the total available free space on this node:
-        long estShardSizeInBytes = Math.max(avgShardSizeInBytes, (long) (totFreeSpace/20.0));
-
-        // Collate predicted (guessed!) disk usage on each path.data:
-        Map<Path,Long> reservedBytes = new HashMap<>();
-        for (IndexShard shard : shards) {
-            Path dataPath = NodeEnvironment.shardStatePathToDataPath(shard.shardPath().getShardStatePath());
-
-            // Remove indices/<index>/<shardID> subdirs from the statePath to get back to the path.data/<lockID>:
-            Long curBytes = reservedBytes.get(dataPath);
-            if (curBytes == null) {
-                curBytes = 0L;
-            }
-            reservedBytes.put(dataPath, curBytes + estShardSizeInBytes);
-        }       
-
-        return reservedBytes;
-    }
-
     public static ShardPath selectNewPathForShard(NodeEnvironment env, ShardId shardId, IndexSettings indexSettings,
                                                   long avgShardSizeInBytes, Map<Path,Integer> dataPathToShardCount) throws IOException {
 
@@ -206,7 +171,6 @@ public final class ShardPath {
             dataPath = env.resolveCustomLocation(indexSettings, shardId);
             statePath = env.nodePaths()[0].resolve(shardId);
         } else {
-
             long totFreeSpace = 0;
             for (NodeEnvironment.NodePath nodePath : env.nodePaths()) {
                 totFreeSpace += nodePath.fileStore.getUsableSpace();
@@ -241,9 +205,7 @@ public final class ShardPath {
             statePath = bestPath.resolve(shardId);
             dataPath = statePath;
         }
-
-        final String indexUUID = indexSettings.getUUID();
-        return new ShardPath(indexSettings.hasCustomDataPath(), dataPath, statePath, indexUUID, shardId);
+        return new ShardPath(indexSettings.hasCustomDataPath(), dataPath, statePath, shardId);
     }
 
     @Override
@@ -258,9 +220,6 @@ public final class ShardPath {
         if (shardId != null ? !shardId.equals(shardPath.shardId) : shardPath.shardId != null) {
             return false;
         }
-        if (indexUUID != null ? !indexUUID.equals(shardPath.indexUUID) : shardPath.indexUUID != null) {
-            return false;
-        }
         if (path != null ? !path.equals(shardPath.path) : shardPath.path != null) {
             return false;
         }
@@ -271,7 +230,6 @@ public final class ShardPath {
     @Override
     public int hashCode() {
         int result = path != null ? path.hashCode() : 0;
-        result = 31 * result + (indexUUID != null ? indexUUID.hashCode() : 0);
         result = 31 * result + (shardId != null ? shardId.hashCode() : 0);
         return result;
     }
@@ -280,7 +238,6 @@ public final class ShardPath {
     public String toString() {
         return "ShardPath{" +
                 "path=" + path +
-                ", indexUUID='" + indexUUID + '\'' +
                 ", shard=" + shardId +
                 '}';
     }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java b/core/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java
index 315371c7286..407f271fd65 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java
@@ -102,7 +102,7 @@ public final class ShardStateMetaData {
         return "version [" + legacyVersion + "], primary [" + primary + "], allocation [" + allocationId + "]";
     }
 
-    public static final MetaDataStateFormat<ShardStateMetaData> FORMAT = new MetaDataStateFormat<ShardStateMetaData>(XContentType.JSON, SHARD_STATE_FILE_PREFIX) {
+    public static final MetaDataStateFormat<ShardStateMetaData> FORMAT = new MetaDataStateFormat<ShardStateMetaData>(XContentType.SMILE, SHARD_STATE_FILE_PREFIX) {
 
         @Override
         protected XContentBuilder newXContentBuilder(XContentType type, OutputStream stream) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
index e057349223d..d11e6734025 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
@@ -128,9 +128,8 @@ final class StoreRecovery {
             assert shardState != IndexShardState.CREATED && shardState != IndexShardState.RECOVERING : "recovery process of " + shardId + " didn't get to post_recovery. shardState [" + shardState + "]";
 
             if (logger.isTraceEnabled()) {
-                StringBuilder sb = new StringBuilder();
-                sb.append("recovery completed from ").append("shard_store").append(", took [").append(timeValueMillis(recoveryState.getTimer().time())).append("]\n");
                 RecoveryState.Index index = recoveryState.getIndex();
+                StringBuilder sb = new StringBuilder();
                 sb.append("    index    : files           [").append(index.totalFileCount()).append("] with total_size [")
                         .append(new ByteSizeValue(index.totalBytes())).append("], took[")
                         .append(TimeValue.timeValueMillis(index.time())).append("]\n");
@@ -142,7 +141,7 @@ final class StoreRecovery {
                         .append(timeValueMillis(recoveryState.getVerifyIndex().checkIndexTime())).append("]\n");
                 sb.append("    translog : number_of_operations [").append(recoveryState.getTranslog().recoveredOperations())
                         .append("], took [").append(TimeValue.timeValueMillis(recoveryState.getTranslog().time())).append("]");
-                logger.trace(sb.toString());
+                logger.trace("recovery completed from [shard_store], took [{}]\n{}", timeValueMillis(recoveryState.getTimer().time()), sb);
             } else if (logger.isDebugEnabled()) {
                 logger.debug("recovery completed from [shard_store], took [{}]", timeValueMillis(recoveryState.getTimer().time()));
             }
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
index e950ebda1b3..865c0eb685b 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.similarity;
 
 import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
 import org.apache.lucene.search.similarities.Similarity;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexModule;
@@ -63,6 +64,10 @@ public final class SimilarityService extends AbstractIndexComponent {
         Map<String, Settings> similaritySettings = this.indexSettings.getSettings().getGroups(IndexModule.SIMILARITY_SETTINGS_PREFIX);
         for (Map.Entry<String, Settings> entry : similaritySettings.entrySet()) {
             String name = entry.getKey();
+            // Starting with v5.0 indices, it should no longer be possible to redefine built-in similarities
+            if(BUILT_IN.containsKey(name) && indexSettings.getIndexVersionCreated().onOrAfter(Version.V_5_0_0_alpha1)) {
+                throw new IllegalArgumentException("Cannot redefine built-in Similarity [" + name + "]");
+            }
             Settings settings = entry.getValue();
             String typeName = settings.get("type");
             if (typeName == null) {
@@ -76,9 +81,16 @@ public final class SimilarityService extends AbstractIndexComponent {
             }
             providers.put(name, factory.apply(name, settings));
         }
-        addSimilarities(similaritySettings, providers, DEFAULTS);
+        for (Map.Entry<String, SimilarityProvider> entry : addSimilarities(similaritySettings, DEFAULTS).entrySet()) {
+            // Avoid overwriting custom providers for indices older that v5.0
+            if (providers.containsKey(entry.getKey()) && indexSettings.getIndexVersionCreated().before(Version.V_5_0_0_alpha1)) {
+                continue;
+            }
+            providers.put(entry.getKey(), entry.getValue());
+        }
         this.similarities = providers;
-        defaultSimilarity = providers.get(SimilarityService.DEFAULT_SIMILARITY).get();
+        defaultSimilarity = (providers.get("default") != null) ? providers.get("default").get()
+                                                              : providers.get(SimilarityService.DEFAULT_SIMILARITY).get();
         // Expert users can configure the base type as being different to default, but out-of-box we use default.
         baseSimilarity = (providers.get("base") != null) ? providers.get("base").get() :
                 defaultSimilarity;
@@ -90,7 +102,9 @@ public final class SimilarityService extends AbstractIndexComponent {
                 defaultSimilarity;
     }
 
-    private void addSimilarities(Map<String, Settings>  similaritySettings, Map<String, SimilarityProvider> providers, Map<String, BiFunction<String, Settings, SimilarityProvider>> similarities)  {
+    private Map<String, SimilarityProvider> addSimilarities(Map<String, Settings>  similaritySettings,
+                                 Map<String, BiFunction<String, Settings, SimilarityProvider>> similarities)  {
+        Map<String, SimilarityProvider> providers = new HashMap<>(similarities.size());
         for (Map.Entry<String, BiFunction<String, Settings, SimilarityProvider>> entry : similarities.entrySet()) {
             String name = entry.getKey();
             BiFunction<String, Settings, SimilarityProvider> factory = entry.getValue();
@@ -100,12 +114,17 @@ public final class SimilarityService extends AbstractIndexComponent {
             }
             providers.put(name, factory.apply(name, settings));
         }
+        return providers;
     }
 
     public SimilarityProvider getSimilarity(String name) {
         return similarities.get(name);
     }
 
+    public SimilarityProvider getDefaultSimilarity() {
+        return similarities.get("default");
+    }
+
     static class PerFieldSimilarity extends PerFieldSimilarityWrapper {
 
         private final Similarity defaultSimilarity;
diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
index 330787a68a3..c15d2cfcdbe 100644
--- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
+++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
@@ -32,9 +32,9 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobMetaData;
diff --git a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
index 06bc6a84a88..584b98cff33 100644
--- a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
+++ b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
@@ -36,7 +36,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
@@ -61,8 +61,9 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim
                 return SimpleFSLockFactory.INSTANCE;
             default:
                 throw new IllegalArgumentException("unrecognized [index.store.fs.fs_lock] \"" + s + "\": must be native or simple");
-        }
-    }, false, Setting.Scope.INDEX);
+        } // can we set on both - node and index level, some nodes might be running on NFS so they might need simple rather than native
+    }, Property.IndexScope, Property.NodeScope);
+
     private final CounterMetric rateLimitingTimeInNanos = new CounterMetric();
     private final ShardPath path;
 
@@ -108,7 +109,8 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim
 
 
     protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
-        final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.DEFAULT.getSettingsKey());
+        final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(),
+            IndexModule.Type.DEFAULT.getSettingsKey());
         if (IndexModule.Type.FS.match(storeType) || IndexModule.Type.DEFAULT.match(storeType)) {
             final FSDirectory open = FSDirectory.open(location, lockFactory); // use lucene defaults
             if (open instanceof MMapDirectory && Constants.WINDOWS == false) {
diff --git a/core/src/main/java/org/elasticsearch/index/store/IndexStore.java b/core/src/main/java/org/elasticsearch/index/store/IndexStore.java
index e98ad7cc6eb..9e01d871765 100644
--- a/core/src/main/java/org/elasticsearch/index/store/IndexStore.java
+++ b/core/src/main/java/org/elasticsearch/index/store/IndexStore.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.store;
 
 import org.apache.lucene.store.StoreRateLimiting;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
@@ -29,8 +30,12 @@ import org.elasticsearch.index.shard.ShardPath;
  *
  */
 public class IndexStore extends AbstractIndexComponent {
-    public static final Setting<IndexRateLimitingType> INDEX_STORE_THROTTLE_TYPE_SETTING = new Setting<>("index.store.throttle.type", "none", IndexRateLimitingType::fromString, true, Setting.Scope.INDEX) ;
-    public static final Setting<ByteSizeValue> INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING = Setting.byteSizeSetting("index.store.throttle.max_bytes_per_sec", new ByteSizeValue(0), true, Setting.Scope.INDEX);
+    public static final Setting<IndexRateLimitingType> INDEX_STORE_THROTTLE_TYPE_SETTING =
+        new Setting<>("index.store.throttle.type", "none", IndexRateLimitingType::fromString,
+            Property.Dynamic, Property.IndexScope);
+    public static final Setting<ByteSizeValue> INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING =
+        Setting.byteSizeSetting("index.store.throttle.max_bytes_per_sec", new ByteSizeValue(0),
+            Property.Dynamic, Property.IndexScope);
 
     protected final IndexStoreConfig indexStoreConfig;
     private final StoreRateLimiting rateLimiting = new StoreRateLimiting();
diff --git a/core/src/main/java/org/elasticsearch/index/store/IndexStoreConfig.java b/core/src/main/java/org/elasticsearch/index/store/IndexStoreConfig.java
index ab7075afa5b..12558bb9554 100644
--- a/core/src/main/java/org/elasticsearch/index/store/IndexStoreConfig.java
+++ b/core/src/main/java/org/elasticsearch/index/store/IndexStoreConfig.java
@@ -22,6 +22,7 @@ import org.apache.lucene.store.StoreRateLimiting;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 
@@ -36,11 +37,15 @@ public class IndexStoreConfig {
     /**
      * Configures the node / cluster level throttle type. See {@link StoreRateLimiting.Type}.
      */
-    public static final Setting<StoreRateLimiting.Type> INDICES_STORE_THROTTLE_TYPE_SETTING = new Setting<>("indices.store.throttle.type", StoreRateLimiting.Type.NONE.name(),StoreRateLimiting.Type::fromString, true, Setting.Scope.CLUSTER);
+    public static final Setting<StoreRateLimiting.Type> INDICES_STORE_THROTTLE_TYPE_SETTING =
+        new Setting<>("indices.store.throttle.type", StoreRateLimiting.Type.NONE.name(),StoreRateLimiting.Type::fromString,
+            Property.Dynamic, Property.NodeScope);
     /**
      * Configures the node / cluster level throttle intensity. The default is <tt>10240 MB</tt>
      */
-    public static final Setting<ByteSizeValue> INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING = Setting.byteSizeSetting("indices.store.throttle.max_bytes_per_sec", new ByteSizeValue(0), true, Setting.Scope.CLUSTER);
+    public static final Setting<ByteSizeValue> INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING =
+        Setting.byteSizeSetting("indices.store.throttle.max_bytes_per_sec", new ByteSizeValue(0),
+            Property.Dynamic, Property.NodeScope);
     private volatile StoreRateLimiting.Type rateLimitingType;
     private volatile ByteSizeValue rateLimitingThrottle;
     private final StoreRateLimiting rateLimiting = new StoreRateLimiting();
diff --git a/core/src/main/java/org/elasticsearch/index/store/Store.java b/core/src/main/java/org/elasticsearch/index/store/Store.java
index 77e7f32f5f5..e0ed3bc98b7 100644
--- a/core/src/main/java/org/elasticsearch/index/store/Store.java
+++ b/core/src/main/java/org/elasticsearch/index/store/Store.java
@@ -49,7 +49,6 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -61,6 +60,7 @@ import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.store.ByteArrayIndexInput;
 import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.Callback;
@@ -90,7 +90,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.zip.Adler32;
 import java.util.zip.CRC32;
 import java.util.zip.Checksum;
 
@@ -124,7 +123,8 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
     static final int VERSION_START = 0;
     static final int VERSION = VERSION_WRITE_THROWABLE;
     static final String CORRUPTED = "corrupted_";
-    public static final Setting<TimeValue> INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING = Setting.timeSetting("index.store.stats_refresh_interval", TimeValue.timeValueSeconds(10), false, Setting.Scope.INDEX);
+    public static final Setting<TimeValue> INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING =
+        Setting.timeSetting("index.store.stats_refresh_interval", TimeValue.timeValueSeconds(10), Property.IndexScope);
 
     private final AtomicBoolean isClosed = new AtomicBoolean(false);
     private final StoreDirectory directory;
@@ -379,7 +379,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
         if (isClosed.compareAndSet(false, true)) {
             // only do this once!
             decRef();
-            logger.debug("store reference count on close: " + refCounter.refCount());
+            logger.debug("store reference count on close: {}", refCounter.refCount());
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/suggest/stats/ShardSuggestMetric.java b/core/src/main/java/org/elasticsearch/index/suggest/stats/ShardSuggestMetric.java
deleted file mode 100644
index 750d7de7b22..00000000000
--- a/core/src/main/java/org/elasticsearch/index/suggest/stats/ShardSuggestMetric.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.suggest.stats;
-
-import org.elasticsearch.common.metrics.CounterMetric;
-import org.elasticsearch.common.metrics.MeanMetric;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- *
- */
-public final class ShardSuggestMetric {
-    private final MeanMetric suggestMetric = new MeanMetric();
-    private final CounterMetric currentMetric = new CounterMetric();
-
-    /**
-     * Called before suggest
-     */
-    public void preSuggest() {
-        currentMetric.inc();
-    }
-
-    /**
-     * Called after suggest
-     * @param tookInNanos time of suggest used in nanos
-     */
-    public void postSuggest(long tookInNanos) {
-        currentMetric.dec();
-        suggestMetric.inc(tookInNanos);
-    }
-
-    /**
-     * @return The current stats
-     */
-    public SuggestStats stats() {
-        return new SuggestStats(suggestMetric.count(), TimeUnit.NANOSECONDS.toMillis(suggestMetric.sum()), currentMetric.count());
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/suggest/stats/SuggestStats.java b/core/src/main/java/org/elasticsearch/index/suggest/stats/SuggestStats.java
deleted file mode 100644
index 1183a64833a..00000000000
--- a/core/src/main/java/org/elasticsearch/index/suggest/stats/SuggestStats.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.suggest.stats;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentBuilderString;
-
-import java.io.IOException;
-
-/**
- * Exposes suggest related statistics.
- */
-public class SuggestStats implements Streamable, ToXContent {
-
-    private long suggestCount;
-    private long suggestTimeInMillis;
-    private long current;
-
-    public SuggestStats() {
-    }
-
-    SuggestStats(long suggestCount, long suggestTimeInMillis, long current) {
-        this.suggestCount = suggestCount;
-        this.suggestTimeInMillis = suggestTimeInMillis;
-        this.current = current;
-    }
-
-    /**
-     * @return The number of times the suggest api has been invoked.
-     */
-    public long getCount() {
-        return suggestCount;
-    }
-
-    /**
-     * @return The total amount of time spend in the suggest api
-     */
-    public long getTimeInMillis() {
-        return suggestTimeInMillis;
-    }
-
-    /**
-     * @return The total amount of time spend in the suggest api
-     */
-    public TimeValue getTime() {
-        return new TimeValue(getTimeInMillis());
-    }
-
-    /**
-     * @return The total amount of active suggest api invocations.
-     */
-    public long getCurrent() {
-        return current;
-    }
-
-    public void add(SuggestStats suggestStats) {
-        if (suggestStats != null) {
-            suggestCount += suggestStats.getCount();
-            suggestTimeInMillis += suggestStats.getTimeInMillis();
-            current += suggestStats.getCurrent();
-        }
-    }
-
-    public static SuggestStats readSuggestStats(StreamInput in) throws IOException {
-        SuggestStats stats = new SuggestStats();
-        stats.readFrom(in);
-        return stats;
-    }
-
-    static final class Fields {
-        static final XContentBuilderString SUGGEST = new XContentBuilderString("suggest");
-        static final XContentBuilderString TOTAL = new XContentBuilderString("total");
-        static final XContentBuilderString TIME = new XContentBuilderString("time");
-        static final XContentBuilderString TIME_IN_MILLIS = new XContentBuilderString("time_in_millis");
-        static final XContentBuilderString CURRENT = new XContentBuilderString("current");
-    }
-
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        suggestCount = in.readVLong();
-        suggestTimeInMillis = in.readVLong();
-        current = in.readVLong();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeVLong(suggestCount);
-        out.writeVLong(suggestTimeInMillis);
-        out.writeVLong(current);
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(Fields.SUGGEST);
-        builder.field(Fields.TOTAL, suggestCount);
-        builder.timeValueField(Fields.TIME_IN_MILLIS, Fields.TIME, suggestTimeInMillis);
-        builder.field(Fields.CURRENT, current);
-        builder.endObject();
-        return builder;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java
index 926ff482248..bd01e7f0183 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java
@@ -30,10 +30,9 @@ import org.apache.lucene.search.QueryCachingPolicy;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Weight;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.ShardCoreKeyMap;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Scope;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.cache.query.QueryCacheStats;
@@ -50,9 +49,9 @@ import java.util.concurrent.ConcurrentHashMap;
 public class IndicesQueryCache extends AbstractComponent implements QueryCache, Closeable {
 
     public static final Setting<ByteSizeValue> INDICES_CACHE_QUERY_SIZE_SETTING = Setting.byteSizeSetting(
-            "indices.queries.cache.size", "10%", false, Scope.CLUSTER);
+            "indices.queries.cache.size", "10%", Property.NodeScope);
     public static final Setting<Integer> INDICES_CACHE_QUERY_COUNT_SETTING = Setting.intSetting(
-            "indices.queries.cache.count", 10000, 1, false, Scope.CLUSTER);
+            "indices.queries.cache.count", 10000, 1, Property.NodeScope);
 
     private final LRUQueryCache cache;
     private final ShardCoreKeyMap shardKeyMap = new ShardCoreKeyMap();
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
index 32b5f55b369..4b4aa4e8df2 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
@@ -34,6 +34,7 @@ import org.elasticsearch.common.cache.RemovalNotification;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
@@ -68,12 +69,12 @@ public final class IndicesRequestCache extends AbstractComponent implements Remo
      * A setting to enable or disable request caching on an index level. Its dynamic by default
      * since we are checking on the cluster state IndexMetaData always.
      */
-    public static final Setting<Boolean> INDEX_CACHE_REQUEST_ENABLED_SETTING = Setting.boolSetting("index.requests.cache.enable",
-        false, true, Setting.Scope.INDEX);
-    public static final Setting<ByteSizeValue> INDICES_CACHE_QUERY_SIZE = Setting.byteSizeSetting("indices.requests.cache.size", "1%",
-        false, Setting.Scope.CLUSTER);
-    public static final Setting<TimeValue> INDICES_CACHE_QUERY_EXPIRE = Setting.positiveTimeSetting("indices.requests.cache.expire",
-        new TimeValue(0), false, Setting.Scope.CLUSTER);
+    public static final Setting<Boolean> INDEX_CACHE_REQUEST_ENABLED_SETTING =
+        Setting.boolSetting("index.requests.cache.enable", true, Property.Dynamic, Property.IndexScope);
+    public static final Setting<ByteSizeValue> INDICES_CACHE_QUERY_SIZE =
+        Setting.byteSizeSetting("indices.requests.cache.size", "1%", Property.NodeScope);
+    public static final Setting<TimeValue> INDICES_CACHE_QUERY_EXPIRE =
+        Setting.positiveTimeSetting("indices.requests.cache.expire", new TimeValue(0), Property.NodeScope);
 
     private final ConcurrentMap<CleanupKey, Boolean> registeredClosedListeners = ConcurrentCollections.newConcurrentMap();
     private final Set<CleanupKey> keysToClean = ConcurrentCollections.newConcurrentSet();
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index 440a11a1904..2719075a5c7 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.indices;
 
+import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.CollectionUtil;
@@ -30,10 +31,11 @@ import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
 import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -47,6 +49,7 @@ import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -61,10 +64,10 @@ import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.cache.request.ShardRequestCache;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.get.GetStats;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.refresh.RefreshStats;
@@ -73,6 +76,7 @@ import org.elasticsearch.index.shard.IllegalIndexShardStateException;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
+import org.elasticsearch.index.shard.IndexingOperationListener;
 import org.elasticsearch.index.shard.IndexingStats;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.IndexStoreConfig;
@@ -87,12 +91,13 @@ import org.elasticsearch.search.query.QueryPhase;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -103,6 +108,7 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.unmodifiableMap;
@@ -115,7 +121,8 @@ import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList;
 public class IndicesService extends AbstractLifecycleComponent<IndicesService> implements Iterable<IndexService>, IndexService.ShardStoreDeleter {
 
     public static final String INDICES_SHARDS_CLOSED_TIMEOUT = "indices.shards_closed_timeout";
-    public static final Setting<TimeValue> INDICES_CACHE_CLEAN_INTERVAL_SETTING = Setting.positiveTimeSetting("indices.cache.cleanup_interval", TimeValue.timeValueMinutes(1), false, Setting.Scope.CLUSTER);
+    public static final Setting<TimeValue> INDICES_CACHE_CLEAN_INTERVAL_SETTING =
+        Setting.positiveTimeSetting("indices.cache.cleanup_interval", TimeValue.timeValueMinutes(1), Property.NodeScope);
     private final PluginsService pluginsService;
     private final NodeEnvironment nodeEnv;
     private final TimeValue shardsClosedTimeout;
@@ -169,7 +176,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
         this.circuitBreakerService = circuitBreakerService;
         this.indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {
             @Override
-            public void onRemoval(ShardId shardId, String fieldName, FieldDataType fieldDataType, boolean wasEvicted, long sizeInBytes) {
+            public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) {
                 assert sizeInBytes >= 0 : "When reducing circuit breaker, it should be adjusted with a number higher or equal to 0 and not [" + sizeInBytes + "]";
                 circuitBreakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(-sizeInBytes);
             }
@@ -185,14 +192,14 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
         ExecutorService indicesStopExecutor = Executors.newFixedThreadPool(5, EsExecutors.daemonThreadFactory("indices_shutdown"));
 
         // Copy indices because we modify it asynchronously in the body of the loop
-        Set<String> indices = new HashSet<>(this.indices.keySet());
+        final Set<Index> indices = this.indices.values().stream().map(s -> s.index()).collect(Collectors.toSet());
         final CountDownLatch latch = new CountDownLatch(indices.size());
-        for (final String index : indices) {
+        for (final Index index : indices) {
             indicesStopExecutor.execute(() -> {
                 try {
                     removeIndex(index, "shutdown", false);
                 } catch (Throwable e) {
-                    logger.warn("failed to remove index on stop [" + index + "]", e);
+                    logger.warn("failed to remove index on stop [{}]", e, index);
                 } finally {
                     latch.countDown();
                 }
@@ -256,13 +263,13 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
         }
 
         Map<Index, List<IndexShardStats>> statsByShard = new HashMap<>();
-        for (IndexService indexService : indices.values()) {
+        for (IndexService indexService : this) {
             for (IndexShard indexShard : indexService) {
                 try {
                     if (indexShard.routingEntry() == null) {
                         continue;
                     }
-                    IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesQueryCache, indexShard, flags), indexShard.commitStats()) });
+                    IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesQueryCache, indexService.cache().getPercolatorQueryCache(), indexShard, flags), indexShard.commitStats()) });
                     if (!statsByShard.containsKey(indexService.index())) {
                         statsByShard.put(indexService.index(), arrayAsArrayList(indexShardStats));
                     } else {
@@ -290,17 +297,8 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
         return indices.values().iterator();
     }
 
-    public boolean hasIndex(String index) {
-        return indices.containsKey(index);
-    }
-
-    /**
-     * Returns an IndexService for the specified index if exists otherwise returns <code>null</code>.
-     *
-     */
-    @Nullable
-    public IndexService indexService(String index) {
-        return indices.get(index);
+    public boolean hasIndex(Index index) {
+        return indices.containsKey(index.getUUID());
     }
 
     /**
@@ -309,33 +307,21 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
      */
     @Nullable
     public IndexService indexService(Index index) {
-        return indexService(index.getName());
-    }
-
-    /**
-     * Returns an IndexService for the specified index if exists otherwise a {@link IndexNotFoundException} is thrown.
-     */
-    public IndexService indexServiceSafe(String index) {
-        IndexService indexService = indexService(index);
-        if (indexService == null) {
-            throw new IndexNotFoundException(index);
-        }
-        return indexService;
+        return indices.get(index.getUUID());
     }
 
     /**
      * Returns an IndexService for the specified index if exists otherwise a {@link IndexNotFoundException} is thrown.
      */
     public IndexService indexServiceSafe(Index index) {
-        IndexService indexService = indexServiceSafe(index.getName());
-        if (indexService.indexUUID().equals(index.getUUID()) == false) {
+        IndexService indexService = indices.get(index.getUUID());
+        if (indexService == null) {
             throw new IndexNotFoundException(index);
         }
+        assert indexService.indexUUID().equals(index.getUUID()) : "uuid mismatch local: " + indexService.indexUUID() + " incoming: " + index.getUUID();
         return indexService;
     }
 
-
-
     /**
      * Creates a new {@link IndexService} for the given metadata.
      * @param indexMetaData the index metadata to create the index for
@@ -343,42 +329,31 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
      * @throws IndexAlreadyExistsException if the index already exists.
      */
     public synchronized IndexService createIndex(final NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData, List<IndexEventListener> builtInListeners) throws IOException {
+
         if (!lifecycle.started()) {
             throw new IllegalStateException("Can't create an index [" + indexMetaData.getIndex() + "], node is closed");
         }
+        if (indexMetaData.getIndexUUID().equals(IndexMetaData.INDEX_UUID_NA_VALUE)) {
+            throw new IllegalArgumentException("index must have a real UUID found value: [" + indexMetaData.getIndexUUID() + "]");
+        }
         final Index index = indexMetaData.getIndex();
-        final Predicate<String> indexNameMatcher = (indexExpression) -> indexNameExpressionResolver.matchesIndex(index.getName(), indexExpression, clusterService.state());
-        final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexNameMatcher, indexScopeSetting);
-        if (indices.containsKey(index.getName())) {
+        if (hasIndex(index)) {
             throw new IndexAlreadyExistsException(index);
         }
-        logger.debug("creating Index [{}], shards [{}]/[{}{}]",
-                indexMetaData.getIndex(),
-                idxSettings.getNumberOfShards(),
-                idxSettings.getNumberOfReplicas(),
-                idxSettings.isShadowReplicaIndex() ? "s" : "");
-
-        final IndexModule indexModule = new IndexModule(idxSettings, indexStoreConfig, analysisRegistry);
-        pluginsService.onIndexModule(indexModule);
-        for (IndexEventListener listener : builtInListeners) {
-            indexModule.addIndexEventListener(listener);
-        }
+        List<IndexEventListener> finalListeners = new ArrayList<>(builtInListeners);
         final IndexEventListener onStoreClose = new IndexEventListener() {
             @Override
             public void onStoreClosed(ShardId shardId) {
                 indicesQueryCache.onClose(shardId);
             }
         };
-        indexModule.addIndexEventListener(onStoreClose);
-        indexModule.addIndexEventListener(oldShardsStats);
-        final IndexEventListener listener = indexModule.freeze();
-        listener.beforeIndexCreated(index, idxSettings.getSettings());
-        final IndexService indexService = indexModule.newIndexService(nodeEnv, this, nodeServicesProvider, indicesQueryCache, mapperRegistry, indicesFieldDataCache, indexingMemoryController);
+        finalListeners.add(onStoreClose);
+        finalListeners.add(oldShardsStats);
+        final IndexService indexService = createIndexService("create index", nodeServicesProvider, indexMetaData, indicesQueryCache, indicesFieldDataCache, finalListeners, indexingMemoryController);
         boolean success = false;
         try {
-            assert indexService.getIndexEventListener() == listener;
-            listener.afterIndexCreated(indexService);
-            indices = newMapBuilder(indices).put(index.getName(), indexService).immutableMap();
+            indexService.getIndexEventListener().afterIndexCreated(indexService);
+            indices = newMapBuilder(indices).put(index.getUUID(), indexService).immutableMap();
             success = true;
             return indexService;
         } finally {
@@ -386,7 +361,54 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
                 indexService.close("plugins_failed", true);
             }
         }
+    }
 
+    /**
+     * This creates a new IndexService without registering it
+     */
+    private synchronized IndexService createIndexService(final String reason, final NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData, IndicesQueryCache indicesQueryCache, IndicesFieldDataCache indicesFieldDataCache, List<IndexEventListener> builtInListeners, IndexingOperationListener... indexingOperationListeners) throws IOException {
+        final Index index = indexMetaData.getIndex();
+        final Predicate<String> indexNameMatcher = (indexExpression) -> indexNameExpressionResolver.matchesIndex(index.getName(), indexExpression, clusterService.state());
+        final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexNameMatcher, indexScopeSetting);
+        logger.debug("creating Index [{}], shards [{}]/[{}{}] - reason [{}]",
+            indexMetaData.getIndex(),
+            idxSettings.getNumberOfShards(),
+            idxSettings.getNumberOfReplicas(),
+            idxSettings.isShadowReplicaIndex() ? "s" : "", reason);
+
+        final IndexModule indexModule = new IndexModule(idxSettings, indexStoreConfig, analysisRegistry);
+        pluginsService.onIndexModule(indexModule);
+        for (IndexEventListener listener : builtInListeners) {
+            indexModule.addIndexEventListener(listener);
+        }
+        final IndexEventListener listener = indexModule.freeze();
+        listener.beforeIndexCreated(index, idxSettings.getSettings());
+        return indexModule.newIndexService(nodeEnv, this, nodeServicesProvider, indicesQueryCache, mapperRegistry, indicesFieldDataCache, indexingOperationListeners);
+    }
+
+    /**
+     * This method verifies that the given {@link IndexMetaData} holds sane values to create an {@link IndexService}. This method will throw an
+     * exception if the creation fails. The created {@link IndexService} will not be registered and will be closed immediately.
+     */
+    public synchronized void verifyIndexMetadata(final NodeServicesProvider nodeServicesProvider, IndexMetaData metaData) throws IOException {
+        final List<Closeable> closeables = new ArrayList<>();
+        try {
+            IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {});
+            closeables.add(indicesFieldDataCache);
+            IndicesQueryCache indicesQueryCache = new IndicesQueryCache(settings);
+            closeables.add(indicesQueryCache);
+            // this will also fail if some plugin fails etc. which is nice since we can verify that early
+            final IndexService service = createIndexService("metadata verification", nodeServicesProvider,
+                metaData, indicesQueryCache, indicesFieldDataCache, Collections.emptyList());
+            for (ObjectCursor<MappingMetaData> typeMapping : metaData.getMappings().values()) {
+                // don't apply the default mapping, it has been applied when the mapping was created
+                service.mapperService().merge(typeMapping.value.type(), typeMapping.value.source(),
+                    MapperService.MergeReason.MAPPING_RECOVERY, true);
+            }
+            closeables.add(() -> service.close("metadata verification", false));
+        } finally {
+            IOUtils.close(closeables);
+        }
     }
 
     /**
@@ -395,22 +417,24 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
      * @param index the index to remove
      * @param reason  the high level reason causing this removal
      */
-    public void removeIndex(String index, String reason) {
+    public void removeIndex(Index index, String reason) {
         removeIndex(index, reason, false);
     }
 
-    private void removeIndex(String index, String reason, boolean delete) {
+    private void removeIndex(Index index, String reason, boolean delete) {
+        final String indexName = index.getName();
         try {
             final IndexService indexService;
             final IndexEventListener listener;
             synchronized (this) {
-                if (indices.containsKey(index) == false) {
+                if (hasIndex(index) == false) {
                     return;
                 }
 
-                logger.debug("[{}] closing ... (reason [{}])", index, reason);
+                logger.debug("[{}] closing ... (reason [{}])", indexName, reason);
                 Map<String, IndexService> newIndices = new HashMap<>(indices);
-                indexService = newIndices.remove(index);
+                indexService = newIndices.remove(index.getUUID());
+                assert indexService != null : "IndexService is null for index: " + index;
                 indices = unmodifiableMap(newIndices);
                 listener = indexService.getIndexEventListener();
             }
@@ -419,9 +443,9 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
             if (delete) {
                 listener.beforeIndexDeleted(indexService);
             }
-            logger.debug("[{}] closing index service (reason [{}])", index, reason);
+            logger.debug("{} closing index service (reason [{}])", index, reason);
             indexService.close(reason, delete);
-            logger.debug("[{}] closed... (reason [{}])", index, reason);
+            logger.debug("{} closed... (reason [{}])", index, reason);
             listener.afterIndexClosed(indexService.index(), indexService.getIndexSettings().getSettings());
             if (delete) {
                 final IndexSettings indexSettings = indexService.getIndexSettings();
@@ -474,12 +498,12 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
      * Deletes the given index. Persistent parts of the index
      * like the shards files, state and transaction logs are removed once all resources are released.
      *
-     * Equivalent to {@link #removeIndex(String, String)} but fires
+     * Equivalent to {@link #removeIndex(Index, String)} but fires
      * different lifecycle events to ensure pending resources of this index are immediately removed.
      * @param index the index to delete
      * @param reason the high level reason causing this delete
      */
-    public void deleteIndex(String index, String reason) throws IOException {
+    public void deleteIndex(Index index, String reason) throws IOException {
         removeIndex(index, reason, true);
     }
 
@@ -505,16 +529,17 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
     public void deleteIndexStore(String reason, IndexMetaData metaData, ClusterState clusterState, boolean closed) throws IOException {
         if (nodeEnv.hasNodeFile()) {
             synchronized (this) {
-                String indexName = metaData.getIndex().getName();
-                if (indices.containsKey(indexName)) {
-                    String localUUid = indices.get(indexName).indexUUID();
-                    throw new IllegalStateException("Can't delete index store for [" + indexName + "] - it's still part of the indices service [" + localUUid + "] [" + metaData.getIndexUUID() + "]");
+                Index index = metaData.getIndex();
+                if (hasIndex(index)) {
+                    String localUUid = indexService(index).indexUUID();
+                    throw new IllegalStateException("Can't delete index store for [" + index.getName() + "] - it's still part of the indices service [" + localUUid + "] [" + metaData.getIndexUUID() + "]");
                 }
-                if (clusterState.metaData().hasIndex(indexName) && (clusterState.nodes().localNode().masterNode() == true)) {
+
+                if (clusterState.metaData().hasIndex(index.getName()) && (clusterState.nodes().localNode().masterNode() == true)) {
                     // we do not delete the store if it is a master eligible node and the index is still in the cluster state
                     // because we want to keep the meta data for indices around even if no shards are left here
-                    final IndexMetaData index = clusterState.metaData().index(indexName);
-                    throw new IllegalStateException("Can't delete closed index store for [" + indexName + "] - it's still part of the cluster state [" + index.getIndexUUID() + "] [" + metaData.getIndexUUID() + "]");
+                    final IndexMetaData idxMeta = clusterState.metaData().index(index.getName());
+                    throw new IllegalStateException("Can't delete closed index store for [" + index.getName() + "] - it's still part of the cluster state [" + idxMeta.getIndexUUID() + "] [" + metaData.getIndexUUID() + "]");
                 }
             }
             final IndexSettings indexSettings = buildIndexSettings(metaData);
@@ -543,7 +568,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
             }
             // this is a pure protection to make sure this index doesn't get re-imported as a dangling index.
             // we should in the future rather write a tombstone rather than wiping the metadata.
-            MetaDataStateFormat.deleteMetaState(nodeEnv.indexPaths(index.getName()));
+            MetaDataStateFormat.deleteMetaState(nodeEnv.indexPaths(index));
         }
     }
 
@@ -554,6 +579,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
      * @param indexSettings the shards index settings.
      * @throws IOException if an IOException occurs
      */
+    @Override
     public void deleteShardStore(String reason, ShardLock lock, IndexSettings indexSettings) throws IOException {
         ShardId shardId = lock.getShardId();
         logger.trace("{} deleting shard reason [{}]", shardId, reason);
@@ -607,7 +633,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
      * @return true if the index can be deleted on this node
      */
     public boolean canDeleteIndexContents(Index index, IndexSettings indexSettings, boolean closed) {
-        final IndexService indexService = this.indices.get(index.getName());
+        final IndexService indexService = indexService(index);
         // Closed indices may be deleted, even if they are on a shared
         // filesystem. Since it is closed we aren't deleting it for relocation
         if (indexSettings.isOnSharedFilesystem() == false || closed) {
@@ -634,14 +660,19 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
      */
     public boolean canDeleteShardContent(ShardId shardId, IndexSettings indexSettings) {
         assert shardId.getIndex().equals(indexSettings.getIndex());
-        final IndexService indexService = this.indices.get(shardId.getIndexName());
+        final IndexService indexService = indexService(shardId.getIndex());
         if (indexSettings.isOnSharedFilesystem() == false) {
-            if (indexService != null && nodeEnv.hasNodeFile()) {
-                return indexService.hasShard(shardId.id()) == false;
-            } else if (nodeEnv.hasNodeFile()) {
-                if (indexSettings.hasCustomDataPath()) {
+           if (nodeEnv.hasNodeFile()) {
+                final boolean isAllocated = indexService != null && indexService.hasShard(shardId.id());
+                if (isAllocated) {
+                    return false; // we are allocated - can't delete the shard
+                } else if (indexSettings.hasCustomDataPath()) {
+                    // lets see if it's on a custom path (return false if the shared doesn't exist)
+                    // we don't need to delete anything that is not there
                     return Files.exists(nodeEnv.resolveCustomLocation(indexSettings, shardId));
                 } else {
+                    // lets see if it's path is available (return false if the shared doesn't exist)
+                    // we don't need to delete anything that is not there
                     return FileSystemUtils.exists(nodeEnv.availableShardPaths(shardId));
                 }
             }
@@ -661,6 +692,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
     /**
      * Adds a pending delete for the given index shard.
      */
+    @Override
     public void addPendingDelete(ShardId shardId, IndexSettings settings) {
         if (shardId == null) {
             throw new IllegalArgumentException("shardId must not be null");
diff --git a/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java b/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
index 7d24d4fa897..237975f2899 100644
--- a/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
+++ b/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
@@ -38,13 +38,12 @@ import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.get.GetStats;
 import org.elasticsearch.index.shard.IndexingStats;
 import org.elasticsearch.index.merge.MergeStats;
-import org.elasticsearch.index.percolator.PercolateStats;
+import org.elasticsearch.index.percolator.PercolatorQueryCacheStats;
 import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.refresh.RefreshStats;
 import org.elasticsearch.index.search.stats.SearchStats;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.store.StoreStats;
-import org.elasticsearch.index.suggest.stats.SuggestStats;
 import org.elasticsearch.search.suggest.completion.CompletionStats;
 
 import java.io.IOException;
@@ -105,8 +104,8 @@ public class NodeIndicesStats implements Streamable, ToXContent {
     }
 
     @Nullable
-    public PercolateStats getPercolate() {
-        return stats.getPercolate();
+    public PercolatorQueryCacheStats getPercolate() {
+        return stats.getPercolatorCache();
     }
 
     @Nullable
@@ -149,11 +148,6 @@ public class NodeIndicesStats implements Streamable, ToXContent {
         return stats.getSegments();
     }
 
-    @Nullable
-    public SuggestStats getSuggest() {
-        return stats.getSuggest();
-    }
-
     @Nullable
     public RecoveryStats getRecoveryStats() {
         return stats.getRecoveryStats();
@@ -172,7 +166,7 @@ public class NodeIndicesStats implements Streamable, ToXContent {
             int entries = in.readVInt();
             statsByShard = new HashMap<>();
             for (int i = 0; i < entries; i++) {
-                Index index = Index.readIndex(in);
+                Index index = new Index(in);
                 int indexShardListSize = in.readVInt();
                 List<IndexShardStats> indexShardStats = new ArrayList<>(indexShardListSize);
                 for (int j = 0; j < indexShardListSize; j++) {
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
index e73396fcd7f..20a1d341cf9 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
@@ -160,15 +160,21 @@ public final class AnalysisModule extends AbstractModule {
     @Override
     protected void configure() {
         try {
-            HunspellService service = new HunspellService(environment.settings(), environment, knownDictionaries);
-            AnalysisRegistry registry = new AnalysisRegistry(service, environment, charFilters, tokenFilters, tokenizers, analyzers);
-            bind(HunspellService.class).toInstance(service);
+            AnalysisRegistry registry = buildRegistry();
+            bind(HunspellService.class).toInstance(registry.getHunspellService());
             bind(AnalysisRegistry.class).toInstance(registry);
         } catch (IOException e) {
             throw new ElasticsearchException("failed to load hunspell service", e);
         }
     }
 
+    /**
+     * Builds an {@link AnalysisRegistry} from the current configuration.
+     */
+    public AnalysisRegistry buildRegistry() throws IOException {
+        return new AnalysisRegistry(new HunspellService(environment.settings(), environment, knownDictionaries), environment, charFilters, tokenFilters, tokenizers, analyzers);
+    }
+
     /**
      * AnalysisProvider is the basic factory interface for registering analysis components like:
      * <ul>
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java b/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
index 5d2fb761842..75c15f09778 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
@@ -26,6 +26,7 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 
@@ -73,9 +74,12 @@ import java.util.function.Function;
  */
 public class HunspellService extends AbstractComponent {
 
-    public final static Setting<Boolean> HUNSPELL_LAZY_LOAD = Setting.boolSetting("indices.analysis.hunspell.dictionary.lazy", Boolean.FALSE, false, Setting.Scope.CLUSTER);
-    public final static Setting<Boolean> HUNSPELL_IGNORE_CASE = Setting.boolSetting("indices.analysis.hunspell.dictionary.ignore_case", Boolean.FALSE, false, Setting.Scope.CLUSTER);
-    public final static Setting<Settings> HUNSPELL_DICTIONARY_OPTIONS = Setting.groupSetting("indices.analysis.hunspell.dictionary.", false, Setting.Scope.CLUSTER);
+    public final static Setting<Boolean> HUNSPELL_LAZY_LOAD =
+        Setting.boolSetting("indices.analysis.hunspell.dictionary.lazy", Boolean.FALSE, Property.NodeScope);
+    public final static Setting<Boolean> HUNSPELL_IGNORE_CASE =
+        Setting.boolSetting("indices.analysis.hunspell.dictionary.ignore_case", Boolean.FALSE, Property.NodeScope);
+    public final static Setting<Settings> HUNSPELL_DICTIONARY_OPTIONS =
+        Setting.groupSetting("indices.analysis.hunspell.dictionary.", Property.NodeScope);
     private final ConcurrentHashMap<String, Dictionary> dictionaries = new ConcurrentHashMap<>();
     private final Map<String, Dictionary> knownDictionaries;
     private final boolean defaultIgnoreCase;
diff --git a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
index 0e1532bc6b3..d2d96092186 100644
--- a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
+++ b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 
@@ -46,15 +47,22 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
 
     private final ConcurrentMap<String, CircuitBreaker> breakers = new ConcurrentHashMap();
 
-    public static final Setting<ByteSizeValue> TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING = Setting.byteSizeSetting("indices.breaker.total.limit", "70%", true, Setting.Scope.CLUSTER);
+    public static final Setting<ByteSizeValue> TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING =
+        Setting.byteSizeSetting("indices.breaker.total.limit", "70%", Property.Dynamic, Property.NodeScope);
 
-    public static final Setting<ByteSizeValue> FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING = Setting.byteSizeSetting("indices.breaker.fielddata.limit", "60%", true, Setting.Scope.CLUSTER);
-    public static final Setting<Double> FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING = Setting.doubleSetting("indices.breaker.fielddata.overhead", 1.03d, 0.0d, true, Setting.Scope.CLUSTER);
-    public static final Setting<CircuitBreaker.Type> FIELDDATA_CIRCUIT_BREAKER_TYPE_SETTING =  new Setting<>("indices.breaker.fielddata.type", "memory", CircuitBreaker.Type::parseValue, false, Setting.Scope.CLUSTER);
+    public static final Setting<ByteSizeValue> FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING =
+        Setting.byteSizeSetting("indices.breaker.fielddata.limit", "60%", Property.Dynamic, Property.NodeScope);
+    public static final Setting<Double> FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING =
+        Setting.doubleSetting("indices.breaker.fielddata.overhead", 1.03d, 0.0d, Property.Dynamic, Property.NodeScope);
+    public static final Setting<CircuitBreaker.Type> FIELDDATA_CIRCUIT_BREAKER_TYPE_SETTING =
+        new Setting<>("indices.breaker.fielddata.type", "memory", CircuitBreaker.Type::parseValue, Property.NodeScope);
 
-    public static final Setting<ByteSizeValue> REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING = Setting.byteSizeSetting("indices.breaker.request.limit", "40%", true, Setting.Scope.CLUSTER);
-    public static final Setting<Double> REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING = Setting.doubleSetting("indices.breaker.request.overhead", 1.0d, 0.0d, true, Setting.Scope.CLUSTER);
-    public static final Setting<CircuitBreaker.Type> REQUEST_CIRCUIT_BREAKER_TYPE_SETTING = new Setting<>("indices.breaker.request.type", "memory", CircuitBreaker.Type::parseValue, false, Setting.Scope.CLUSTER);
+    public static final Setting<ByteSizeValue> REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING =
+        Setting.byteSizeSetting("indices.breaker.request.limit", "40%", Property.Dynamic, Property.NodeScope);
+    public static final Setting<Double> REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING =
+        Setting.doubleSetting("indices.breaker.request.overhead", 1.0d, 0.0d, Property.Dynamic, Property.NodeScope);
+    public static final Setting<CircuitBreaker.Type> REQUEST_CIRCUIT_BREAKER_TYPE_SETTING =
+        new Setting<>("indices.breaker.request.type", "memory", CircuitBreaker.Type::parseValue, Property.NodeScope);
 
 
 
diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index 7998afb7656..82f1466bf13 100644
--- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -19,10 +19,8 @@
 
 package org.elasticsearch.indices.cluster;
 
-import com.carrotsearch.hppc.IntHashSet;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction;
@@ -35,9 +33,11 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RestoreSource;
+import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -46,6 +46,7 @@ import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexShardAlreadyExistsException;
@@ -70,9 +71,11 @@ import org.elasticsearch.snapshots.RestoreService;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 
 /**
@@ -89,7 +92,8 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
     private final NodeMappingRefreshAction nodeMappingRefreshAction;
     private final NodeServicesProvider nodeServicesProvider;
 
-    private static final ShardStateAction.Listener SHARD_STATE_ACTION_LISTENER = new ShardStateAction.Listener() {};
+    private static final ShardStateAction.Listener SHARD_STATE_ACTION_LISTENER = new ShardStateAction.Listener() {
+    };
 
     // a list of shards that failed during recovery
     // we keep track of these shards in order to prevent repeated recovery of these shards on each cluster state update
@@ -157,13 +161,13 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
             // TODO: this feels a bit hacky here, a block disables state persistence, and then we clean the allocated shards, maybe another flag in blocks?
             if (event.state().blocks().disableStatePersistence()) {
                 for (IndexService indexService : indicesService) {
-                    String index = indexService.index().getName();
+                    Index index = indexService.index();
                     for (Integer shardId : indexService.shardIds()) {
-                        logger.debug("[{}][{}] removing shard (disabled block persistence)", index, shardId);
+                        logger.debug("{}[{}] removing shard (disabled block persistence)", index, shardId);
                         try {
                             indexService.removeShard(shardId, "removing shard (disabled block persistence)");
                         } catch (Throwable e) {
-                            logger.warn("[{}] failed to remove shard (disabled block persistence)", e, index);
+                            logger.warn("{} failed to remove shard (disabled block persistence)", e, index);
                         }
                     }
                     removeIndex(index, "cleaning index (disabled block persistence)");
@@ -173,41 +177,44 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
 
             cleanFailedShards(event);
 
+            // cleaning up indices that are completely deleted so we won't need to worry about them
+            // when checking for shards
             applyDeletedIndices(event);
+            applyDeletedShards(event);
+            // call after deleted shards so indices with no shards will be cleaned
+            applyCleanedIndices(event);
+            // make sure that newly created shards use the latest meta data
+            applyIndexMetaData(event);
             applyNewIndices(event);
+            // apply mappings also updates new indices. TODO: make new indices good to begin with
             applyMappings(event);
             applyNewOrUpdatedShards(event);
-            applyDeletedShards(event);
-            applyCleanedIndices(event);
-            applySettings(event);
         }
     }
 
-    private void applyCleanedIndices(final ClusterChangedEvent event) {
-        // handle closed indices, since they are not allocated on a node once they are closed
-        // so applyDeletedIndices might not take them into account
-        for (IndexService indexService : indicesService) {
-            String index = indexService.index().getName();
-            IndexMetaData indexMetaData = event.state().metaData().index(index);
-            if (indexMetaData != null && indexMetaData.getState() == IndexMetaData.State.CLOSE) {
-                for (Integer shardId : indexService.shardIds()) {
-                    logger.debug("[{}][{}] removing shard (index is closed)", index, shardId);
-                    try {
-                        indexService.removeShard(shardId, "removing shard (index is closed)");
-                    } catch (Throwable e) {
-                        logger.warn("[{}] failed to remove shard (index is closed)", e, index);
-                    }
-                }
-            }
+    private void cleanFailedShards(final ClusterChangedEvent event) {
+        RoutingNodes.RoutingNodeIterator routingNode = event.state().getRoutingNodes().routingNodeIter(event.state().nodes().localNodeId());
+        if (routingNode == null) {
+            failedShards.clear();
+            return;
         }
-        for (IndexService indexService : indicesService) {
-            String index = indexService.index().getName();
-            if (indexService.shardIds().isEmpty()) {
-                if (logger.isDebugEnabled()) {
-                    logger.debug("[{}] cleaning index (no shards allocated)", index);
-                }
-                // clean the index
-                removeIndex(index, "removing index (no shards allocated)");
+        RoutingTable routingTable = event.state().routingTable();
+        for (Iterator<Map.Entry<ShardId, ShardRouting>> iterator = failedShards.entrySet().iterator(); iterator.hasNext(); ) {
+            Map.Entry<ShardId, ShardRouting> entry = iterator.next();
+            ShardId failedShardId = entry.getKey();
+            ShardRouting failedShardRouting = entry.getValue();
+            IndexRoutingTable indexRoutingTable = routingTable.index(failedShardId.getIndex());
+            if (indexRoutingTable == null) {
+                iterator.remove();
+                continue;
+            }
+            IndexShardRoutingTable shardRoutingTable = indexRoutingTable.shard(failedShardId.id());
+            if (shardRoutingTable == null) {
+                iterator.remove();
+                continue;
+            }
+            if (shardRoutingTable.assignedShards().stream().noneMatch(shr -> shr.isSameAllocation(failedShardRouting))) {
+                iterator.remove();
             }
         }
     }
@@ -217,17 +224,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
         final String localNodeId = event.state().nodes().localNodeId();
         assert localNodeId != null;
 
-        for (IndexService indexService : indicesService) {
-            IndexMetaData indexMetaData = event.state().metaData().index(indexService.index().getName());
-            if (indexMetaData != null) {
-                if (!indexMetaData.isSameUUID(indexService.indexUUID())) {
-                    logger.debug("[{}] mismatch on index UUIDs between cluster state and local state, cleaning the index so it will be recreated", indexMetaData.getIndex());
-                    deleteIndex(indexMetaData.getIndex().getName(), "mismatch on index UUIDs between cluster state and local state, cleaning the index so it will be recreated");
-                }
-            }
-        }
-
-        for (String index : event.indicesDeleted()) {
+        for (Index index : event.indicesDeleted()) {
             if (logger.isDebugEnabled()) {
                 logger.debug("[{}] cleaning index, no longer part of the metadata", index);
             }
@@ -237,8 +234,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
                 indexSettings = idxService.getIndexSettings();
                 deleteIndex(index, "index no longer part of the metadata");
             } else {
-                final IndexMetaData metaData = previousState.metaData().index(index);
-                assert metaData != null;
+                final IndexMetaData metaData = previousState.metaData().getIndexSafe(index);
                 indexSettings = new IndexSettings(metaData, settings);
                 indicesService.deleteClosedIndex("closed index no longer part of the metadata", metaData, event.state());
             }
@@ -249,7 +245,17 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
             }
         }
 
-
+        for (IndexService indexService : indicesService) {
+            IndexMetaData indexMetaData = event.state().metaData().index(indexService.index());
+            if (indexMetaData == null) {
+                assert false : "index" + indexService.index() + " exists locally, doesn't have a metadata but is not part "
+                    + " of the delete index list. \nprevious state: " + event.previousState().prettyPrint()
+                    + "\n current state:\n" + event.state().prettyPrint();
+                logger.warn("[{}] isn't part of metadata but is part of in memory structures. removing",
+                    indexService.index());
+                deleteIndex(indexService.index(), "isn't part of metadata (explicit check)");
+            }
+        }
     }
 
     private void applyDeletedShards(final ClusterChangedEvent event) {
@@ -257,40 +263,103 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
         if (routingNode == null) {
             return;
         }
-        IntHashSet newShardIds = new IntHashSet();
+        Set<String> newShardAllocationIds = new HashSet<>();
         for (IndexService indexService : indicesService) {
-            String indexName = indexService.index().getName();
-            IndexMetaData indexMetaData = event.state().metaData().index(indexName);
-            if (indexMetaData == null) {
-                continue;
-            }
+            Index index = indexService.index();
+            IndexMetaData indexMetaData = event.state().metaData().index(index);
+            assert indexMetaData != null : "local index doesn't have metadata, should have been cleaned up by applyDeletedIndices: " + index;
             // now, go over and delete shards that needs to get deleted
-            newShardIds.clear();
+            newShardAllocationIds.clear();
             for (ShardRouting shard : routingNode) {
-                if (shard.index().getName().equals(indexName)) {
-                    newShardIds.add(shard.id());
+                if (shard.index().equals(index)) {
+                    // use the allocation id and not object so we won't be influence by relocation targets
+                    newShardAllocationIds.add(shard.allocationId().getId());
                 }
             }
-            for (Integer existingShardId : indexService.shardIds()) {
-                if (!newShardIds.contains(existingShardId)) {
+            for (IndexShard existingShard : indexService) {
+                if (newShardAllocationIds.contains(existingShard.routingEntry().allocationId().getId()) == false) {
                     if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
                         if (logger.isDebugEnabled()) {
-                            logger.debug("[{}][{}] removing shard (index is closed)", indexName, existingShardId);
+                            logger.debug("{} removing shard (index is closed)", existingShard.shardId());
                         }
-                        indexService.removeShard(existingShardId, "removing shard (index is closed)");
+                        indexService.removeShard(existingShard.shardId().id(), "removing shard (index is closed)");
                     } else {
                         // we can just remove the shard, without cleaning it locally, since we will clean it
                         // when all shards are allocated in the IndicesStore
                         if (logger.isDebugEnabled()) {
-                            logger.debug("[{}][{}] removing shard (not allocated)", indexName, existingShardId);
+                            logger.debug("{} removing shard (not allocated)", existingShard.shardId());
                         }
-                        indexService.removeShard(existingShardId, "removing shard (not allocated)");
+                        indexService.removeShard(existingShard.shardId().id(), "removing shard (not allocated)");
                     }
                 }
             }
         }
     }
 
+    private void applyCleanedIndices(final ClusterChangedEvent event) {
+        // handle closed indices, since they are not allocated on a node once they are closed
+        // so applyDeletedIndices might not take them into account
+        for (IndexService indexService : indicesService) {
+            Index index = indexService.index();
+            IndexMetaData indexMetaData = event.state().metaData().index(index);
+            if (indexMetaData != null && indexMetaData.getState() == IndexMetaData.State.CLOSE) {
+                for (Integer shardId : indexService.shardIds()) {
+                    logger.debug("{}[{}] removing shard (index is closed)", index, shardId);
+                    try {
+                        indexService.removeShard(shardId, "removing shard (index is closed)");
+                    } catch (Throwable e) {
+                        logger.warn("{} failed to remove shard (index is closed)", e, index);
+                    }
+                }
+            }
+        }
+
+        final Set<Index> hasAllocations = new HashSet<>();
+        final RoutingNode node = event.state().getRoutingNodes().node(event.state().nodes().localNodeId());
+        // if no shards are allocated ie. if this node is a master-only node it can return nul
+        if (node != null) {
+            for (ShardRouting routing : node) {
+                hasAllocations.add(routing.index());
+            }
+        }
+        for (IndexService indexService : indicesService) {
+            Index index = indexService.index();
+            if (hasAllocations.contains(index) == false) {
+                assert indexService.shardIds().isEmpty() :
+                    "no locally assigned shards, but index wasn't emptied by applyDeletedShards."
+                        + " index " + index + ", shards: " + indexService.shardIds();
+                if (logger.isDebugEnabled()) {
+                    logger.debug("{} cleaning index (no shards allocated)", index);
+                }
+                // clean the index
+                removeIndex(index, "removing index (no shards allocated)");
+            }
+        }
+    }
+
+    private void applyIndexMetaData(ClusterChangedEvent event) {
+        if (!event.metaDataChanged()) {
+            return;
+        }
+        for (IndexMetaData indexMetaData : event.state().metaData()) {
+            if (!indicesService.hasIndex(indexMetaData.getIndex())) {
+                // we only create / update here
+                continue;
+            }
+            // if the index meta data didn't change, no need check for refreshed settings
+            if (!event.indexMetaDataChanged(indexMetaData)) {
+                continue;
+            }
+            Index index = indexMetaData.getIndex();
+            IndexService indexService = indicesService.indexService(index);
+            if (indexService == null) {
+                // already deleted on us, ignore it
+                continue;
+            }
+            indexService.updateMetaData(indexMetaData);
+        }
+    }
+
     private void applyNewIndices(final ClusterChangedEvent event) {
         // we only create indices for shards that are allocated
         RoutingNodes.RoutingNodeIterator routingNode = event.state().getRoutingNodes().routingNodeIter(event.state().nodes().localNodeId());
@@ -298,8 +367,8 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
             return;
         }
         for (ShardRouting shard : routingNode) {
-            if (!indicesService.hasIndex(shard.getIndexName())) {
-                final IndexMetaData indexMetaData = event.state().metaData().index(shard.index());
+            if (!indicesService.hasIndex(shard.index())) {
+                final IndexMetaData indexMetaData = event.state().metaData().getIndexSafe(shard.index());
                 if (logger.isDebugEnabled()) {
                     logger.debug("[{}] creating index", indexMetaData.getIndex());
                 }
@@ -312,39 +381,15 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
         }
     }
 
-    private void applySettings(ClusterChangedEvent event) {
-        if (!event.metaDataChanged()) {
-            return;
-        }
-        for (IndexMetaData indexMetaData : event.state().metaData()) {
-            if (!indicesService.hasIndex(indexMetaData.getIndex().getName())) {
-                // we only create / update here
-                continue;
-            }
-            // if the index meta data didn't change, no need check for refreshed settings
-            if (!event.indexMetaDataChanged(indexMetaData)) {
-                continue;
-            }
-            String index = indexMetaData.getIndex().getName();
-            IndexService indexService = indicesService.indexService(index);
-            if (indexService == null) {
-                // already deleted on us, ignore it
-                continue;
-            }
-            indexService.updateMetaData(indexMetaData);
-        }
-    }
-
-
     private void applyMappings(ClusterChangedEvent event) {
         // go over and update mappings
         for (IndexMetaData indexMetaData : event.state().metaData()) {
-            if (!indicesService.hasIndex(indexMetaData.getIndex().getName())) {
+            Index index = indexMetaData.getIndex();
+            if (!indicesService.hasIndex(index)) {
                 // we only create / update here
                 continue;
             }
             boolean requireRefresh = false;
-            String index = indexMetaData.getIndex().getName();
             IndexService indexService = indicesService.indexService(index);
             if (indexService == null) {
                 // got deleted on us, ignore (closing the node)
@@ -357,12 +402,12 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
                     MappingMetaData mappingMd = cursor.value;
                     String mappingType = mappingMd.type();
                     CompressedXContent mappingSource = mappingMd.source();
-                    requireRefresh |= processMapping(index, mapperService, mappingType, mappingSource);
+                    requireRefresh |= processMapping(index.getName(), mapperService, mappingType, mappingSource);
                 }
                 if (requireRefresh && sendRefreshMapping) {
                     nodeMappingRefreshAction.nodeMappingRefresh(event.state(),
-                            new NodeMappingRefreshAction.NodeMappingRefreshRequest(index, indexMetaData.getIndexUUID(),
-                                    event.state().nodes().localNodeId())
+                        new NodeMappingRefreshAction.NodeMappingRefreshRequest(index.getName(), indexMetaData.getIndexUUID(),
+                            event.state().nodes().localNodeId())
                     );
                 }
             } catch (Throwable t) {
@@ -426,14 +471,13 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
         for (final ShardRouting shardRouting : routingNode) {
             final IndexService indexService = indicesService.indexService(shardRouting.index());
             if (indexService == null) {
-                // got deleted on us, ignore
+                // creation failed for some reasons
+                assert failedShards.containsKey(shardRouting.shardId()) :
+                    "index has local allocation but is not created by applyNewIndices and is not failed " + shardRouting;
                 continue;
             }
             final IndexMetaData indexMetaData = event.state().metaData().index(shardRouting.index());
-            if (indexMetaData == null) {
-                // the index got deleted on the metadata, we will clean it later in the apply deleted method call
-                continue;
-            }
+            assert indexMetaData != null : "index has local allocation but no meta data. " + shardRouting.index();
 
             final int shardId = shardRouting.id();
 
@@ -458,12 +502,10 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
                 // for example: a shard that recovers from one node and now needs to recover to another node,
                 //              or a replica allocated and then allocating a primary because the primary failed on another node
                 boolean shardHasBeenRemoved = false;
-                if (currentRoutingEntry.isSameAllocation(shardRouting) == false) {
-                    logger.debug("[{}][{}] removing shard (different instance of it allocated on this node, current [{}], global [{}])", shardRouting.index(), shardRouting.id(), currentRoutingEntry, shardRouting);
-                    // closing the shard will also cancel any ongoing recovery.
-                    indexService.removeShard(shardRouting.id(), "removing shard (different instance of it allocated on this node)");
-                    shardHasBeenRemoved = true;
-                } else if (isPeerRecovery(shardRouting)) {
+                assert currentRoutingEntry.isSameAllocation(shardRouting) :
+                    "local shard has a different allocation id but wasn't cleaning by applyDeletedShards. "
+                        + "cluster state: " + shardRouting + " local: " + currentRoutingEntry;
+                if (isPeerRecovery(shardRouting)) {
                     final DiscoveryNode sourceNode = findSourceNodeForPeerRecovery(routingTable, nodes, shardRouting);
                     // check if there is an existing recovery going, and if so, and the source node is not the same, cancel the recovery to restart it
                     if (recoveryTargetService.cancelRecoveriesForShard(indexShard.shardId(), "recovery source node changed", status -> !status.sourceNode().equals(sourceNode))) {
@@ -477,7 +519,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
                 if (shardHasBeenRemoved == false) {
                     // shadow replicas do not support primary promotion. The master would reinitialize the shard, giving it a new allocation, meaning we should be there.
                     assert (shardRouting.primary() && currentRoutingEntry.primary() == false) == false || indexShard.allowsPrimaryPromotion() :
-                            "shard for doesn't support primary promotion but master promoted it with changing allocation. New routing " + shardRouting + ", current routing " + currentRoutingEntry;
+                        "shard for doesn't support primary promotion but master promoted it with changing allocation. New routing " + shardRouting + ", current routing " + currentRoutingEntry;
                     try {
                         indexShard.updateRoutingEntry(shardRouting, event.state().blocks().disableStatePersistence() == false);
                     } catch (Throwable e) {
@@ -487,44 +529,12 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
             }
 
             if (shardRouting.initializing()) {
-                applyInitializingShard(event.state(), indexMetaData, shardRouting);
+                applyInitializingShard(event.state(), indexMetaData, indexService, shardRouting);
             }
         }
     }
 
-    private void cleanFailedShards(final ClusterChangedEvent event) {
-        RoutingNodes.RoutingNodeIterator routingNode = event.state().getRoutingNodes().routingNodeIter(event.state().nodes().localNodeId());
-        if (routingNode == null) {
-            failedShards.clear();
-            return;
-        }
-        RoutingTable routingTable = event.state().routingTable();
-        for (Iterator<Map.Entry<ShardId, ShardRouting>> iterator = failedShards.entrySet().iterator(); iterator.hasNext(); ) {
-            Map.Entry<ShardId, ShardRouting> entry = iterator.next();
-            ShardId failedShardId = entry.getKey();
-            ShardRouting failedShardRouting = entry.getValue();
-            IndexRoutingTable indexRoutingTable = routingTable.index(failedShardId.getIndex());
-            if (indexRoutingTable == null) {
-                iterator.remove();
-                continue;
-            }
-            IndexShardRoutingTable shardRoutingTable = indexRoutingTable.shard(failedShardId.id());
-            if (shardRoutingTable == null) {
-                iterator.remove();
-                continue;
-            }
-            if (shardRoutingTable.assignedShards().stream().noneMatch(shr -> shr.isSameAllocation(failedShardRouting))) {
-                iterator.remove();
-            }
-        }
-    }
-
-    private void applyInitializingShard(final ClusterState state, final IndexMetaData indexMetaData, final ShardRouting shardRouting) {
-        final IndexService indexService = indicesService.indexService(shardRouting.index());
-        if (indexService == null) {
-            // got deleted on us, ignore
-            return;
-        }
+    private void applyInitializingShard(final ClusterState state, final IndexMetaData indexMetaData, IndexService indexService, final ShardRouting shardRouting) {
         final RoutingTable routingTable = state.routingTable();
         final DiscoveryNodes nodes = state.getNodes();
         final int shardId = shardRouting.id();
@@ -537,7 +547,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
                 // we managed to tell the master we started), mark us as started
                 if (logger.isTraceEnabled()) {
                     logger.trace("{} master marked shard as initializing, but shard has state [{}], resending shard started to {}",
-                            indexShard.shardId(), indexShard.state(), nodes.masterNode());
+                        indexShard.shardId(), indexShard.state(), nodes.masterNode());
                 }
                 if (nodes.masterNode() != null) {
                     shardStateAction.shardStarted(shardRouting,
@@ -618,8 +628,8 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
             assert indexShard.routingEntry().equals(shardRouting); // should have already be done before
             // recover from filesystem store
             final RecoveryState recoveryState = new RecoveryState(indexShard.shardId(), shardRouting.primary(),
-                    RecoveryState.Type.STORE,
-                    nodes.localNode(), nodes.localNode());
+                RecoveryState.Type.STORE,
+                nodes.localNode(), nodes.localNode());
             indexShard.markAsRecovering("from store", recoveryState); // mark the shard as recovering on the cluster state thread
             threadPool.generic().execute(() -> {
                 try {
@@ -634,7 +644,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
         } else {
             // recover from a restore
             final RecoveryState recoveryState = new RecoveryState(indexShard.shardId(), shardRouting.primary(),
-                    RecoveryState.Type.SNAPSHOT, shardRouting.restoreSource(), nodes.localNode());
+                RecoveryState.Type.SNAPSHOT, shardRouting.restoreSource(), nodes.localNode());
             indexShard.markAsRecovering("from snapshot", recoveryState); // mark the shard as recovering on the cluster state thread
             threadPool.generic().execute(() -> {
                 final ShardId sId = indexShard.shardId();
@@ -727,7 +737,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
         }
     }
 
-    private void removeIndex(String index, String reason) {
+    private void removeIndex(Index index, String reason) {
         try {
             indicesService.removeIndex(index, reason);
         } catch (Throwable e) {
@@ -735,7 +745,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
         }
     }
 
-    private void deleteIndex(String index, String reason) {
+    private void deleteIndex(Index index, String reason) {
         try {
             indicesService.deleteIndex(index, reason);
         } catch (Throwable e) {
@@ -774,7 +784,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
     private class FailedShardHandler implements Callback<IndexShard.ShardFailure> {
         @Override
         public void handle(final IndexShard.ShardFailure shardFailure) {
-            final IndexService indexService = indicesService.indexService(shardFailure.routing.shardId().getIndex().getName());
+            final IndexService indexService = indicesService.indexService(shardFailure.routing.shardId().getIndex());
             final ShardRouting shardRouting = shardFailure.routing;
             threadPool.generic().execute(() -> {
                 synchronized (mutex) {
diff --git a/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java b/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java
index 0a3f063dfcc..4f7d482f8a9 100644
--- a/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java
+++ b/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java
@@ -34,11 +34,11 @@ import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.AtomicFieldData;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.shard.ShardId;
@@ -52,7 +52,8 @@ import java.util.function.ToLongBiFunction;
  */
 public class IndicesFieldDataCache extends AbstractComponent implements RemovalListener<IndicesFieldDataCache.Key, Accountable>, Releasable{
 
-    public static final Setting<ByteSizeValue> INDICES_FIELDDATA_CACHE_SIZE_KEY = Setting.byteSizeSetting("indices.fielddata.cache.size", new ByteSizeValue(-1), false, Setting.Scope.CLUSTER);
+    public static final Setting<ByteSizeValue> INDICES_FIELDDATA_CACHE_SIZE_KEY =
+        Setting.byteSizeSetting("indices.fielddata.cache.size", new ByteSizeValue(-1), Property.NodeScope);
     private final IndexFieldDataCache.Listener indicesFieldDataCacheListener;
     private final Cache<Key, Accountable> cache;
 
@@ -73,8 +74,8 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
         cache.invalidateAll();
     }
 
-    public IndexFieldDataCache buildIndexFieldDataCache(IndexFieldDataCache.Listener listener, Index index, String fieldName, FieldDataType fieldDataType) {
-        return new IndexFieldCache(logger, cache, index, fieldName, fieldDataType, indicesFieldDataCacheListener, listener);
+    public IndexFieldDataCache buildIndexFieldDataCache(IndexFieldDataCache.Listener listener, Index index, String fieldName) {
+        return new IndexFieldCache(logger, cache, index, fieldName, indicesFieldDataCacheListener, listener);
     }
 
     public Cache<Key, Accountable> getCache() {
@@ -89,7 +90,7 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
         final Accountable value = notification.getValue();
         for (IndexFieldDataCache.Listener listener : key.listeners) {
             try {
-                listener.onRemoval(key.shardId, indexCache.fieldName, indexCache.fieldDataType, notification.getRemovalReason() == RemovalNotification.RemovalReason.EVICTED, value.ramBytesUsed());
+                listener.onRemoval(key.shardId, indexCache.fieldName, notification.getRemovalReason() == RemovalNotification.RemovalReason.EVICTED, value.ramBytesUsed());
             } catch (Throwable e) {
                 // load anyway since listeners should not throw exceptions
                 logger.error("Failed to call listener on field data cache unloading", e);
@@ -112,16 +113,14 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
         private final ESLogger logger;
         final Index index;
         final String fieldName;
-        final FieldDataType fieldDataType;
         private final Cache<Key, Accountable> cache;
         private final Listener[] listeners;
 
-        IndexFieldCache(ESLogger logger,final Cache<Key, Accountable> cache, Index index, String fieldName, FieldDataType fieldDataType, Listener... listeners) {
+        IndexFieldCache(ESLogger logger,final Cache<Key, Accountable> cache, Index index, String fieldName, Listener... listeners) {
             this.logger = logger;
             this.listeners = listeners;
             this.index = index;
             this.fieldName = fieldName;
-            this.fieldDataType = fieldDataType;
             this.cache = cache;
         }
 
@@ -138,7 +137,7 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
                 final AtomicFieldData fieldData = indexFieldData.loadDirect(context);
                 for (Listener listener : k.listeners) {
                     try {
-                        listener.onCache(shardId, fieldName, fieldDataType, fieldData);
+                        listener.onCache(shardId, fieldName, fieldData);
                     } catch (Throwable e) {
                         // load anyway since listeners should not throw exceptions
                         logger.error("Failed to call listener on atomic field data loading", e);
@@ -162,7 +161,7 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
                 final Accountable ifd = (Accountable) indexFieldData.localGlobalDirect(indexReader);
                 for (Listener listener : k.listeners) {
                     try {
-                        listener.onCache(shardId, fieldName, fieldDataType, ifd);
+                        listener.onCache(shardId, fieldName, ifd);
                     } catch (Throwable e) {
                         // load anyway since listeners should not throw exceptions
                         logger.error("Failed to call listener on global ordinals loading", e);
diff --git a/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCacheListener.java b/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCacheListener.java
index c37cf6def79..1995bb2dfb8 100644
--- a/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCacheListener.java
+++ b/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCacheListener.java
@@ -22,7 +22,6 @@ package org.elasticsearch.indices.fielddata.cache;
 import org.apache.lucene.util.Accountable;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
@@ -43,11 +42,11 @@ public class IndicesFieldDataCacheListener implements IndexFieldDataCache.Listen
     }
 
     @Override
-    public void onCache(ShardId shardId, String fieldName, FieldDataType fieldDataType, Accountable fieldData) {
+    public void onCache(ShardId shardId, String fieldName, Accountable fieldData) {
     }
 
     @Override
-    public void onRemoval(ShardId shardId, String fieldName, FieldDataType fieldDataType, boolean wasEvicted, long sizeInBytes) {
+    public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) {
         assert sizeInBytes >= 0 : "When reducing circuit breaker, it should be adjusted with a number higher or equal to 0 and not [" + sizeInBytes + "]";
         circuitBreakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(-sizeInBytes);
     }
diff --git a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
index 6eb7c88a2a4..b1d7af7ff9c 100644
--- a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
+++ b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.flush.SyncedFlushResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -31,6 +30,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
@@ -39,6 +39,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.CountDown;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
@@ -111,15 +112,15 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
      */
     public void attemptSyncedFlush(final String[] aliasesOrIndices, IndicesOptions indicesOptions, final ActionListener<SyncedFlushResponse> listener) {
         final ClusterState state = clusterService.state();
-        final String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, indicesOptions, aliasesOrIndices);
+        final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, indicesOptions, aliasesOrIndices);
         final Map<String, List<ShardsSyncedFlushResult>> results = ConcurrentCollections.newConcurrentMap();
         int totalNumberOfShards = 0;
         int numberOfShards = 0;
-        for (String index : concreteIndices) {
-            final IndexMetaData indexMetaData = state.metaData().index(index);
+        for (Index index : concreteIndices) {
+            final IndexMetaData indexMetaData = state.metaData().getIndexSafe(index);
             totalNumberOfShards += indexMetaData.getTotalNumberOfShards();
             numberOfShards += indexMetaData.getNumberOfShards();
-            results.put(index, Collections.synchronizedList(new ArrayList<>()));
+            results.put(index.getName(), Collections.synchronizedList(new ArrayList<>()));
 
         }
         if (numberOfShards == 0) {
@@ -129,8 +130,9 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
         final int finalTotalNumberOfShards = totalNumberOfShards;
         final CountDown countDown = new CountDown(numberOfShards);
 
-        for (final String index : concreteIndices) {
-            final IndexMetaData indexMetaData = state.metaData().index(index);
+        for (final Index concreteIndex : concreteIndices) {
+            final String index = concreteIndex.getName();
+            final IndexMetaData indexMetaData = state.metaData().getIndexSafe(concreteIndex);
             final int indexNumberOfShards = indexMetaData.getNumberOfShards();
             for (int shard = 0; shard < indexNumberOfShards; shard++) {
                 final ShardId shardId = new ShardId(indexMetaData.getIndex(), shard);
@@ -240,7 +242,7 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
     final IndexShardRoutingTable getShardRoutingTable(ShardId shardId, ClusterState state) {
         final IndexRoutingTable indexRoutingTable = state.routingTable().index(shardId.getIndexName());
         if (indexRoutingTable == null) {
-            IndexMetaData index = state.getMetaData().index(shardId.getIndexName());
+            IndexMetaData index = state.getMetaData().index(shardId.getIndex());
             if (index != null && index.getState() == IndexMetaData.State.CLOSE) {
                 throw new IndexClosedException(shardId.getIndex());
             }
@@ -309,7 +311,7 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
             }
             final Engine.CommitId expectedCommitId = expectedCommitIds.get(shard.currentNodeId());
             if (expectedCommitId == null) {
-                logger.trace("{} can't resolve expected commit id for {}, skipping for sync id [{}]. shard routing {}", shardId, syncId, shard);
+                logger.trace("{} can't resolve expected commit id for current node, skipping for sync id [{}]. shard routing {}", shardId, syncId, shard);
                 results.put(shard, new ShardSyncedFlushResponse("no commit id from pre-sync flush"));
                 contDownAndSendResponseIfDone(syncId, shards, shardId, totalShards, listener, countDown, results);
                 continue;
diff --git a/core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java b/core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java
index a9e90884a68..b0b212d2ab4 100644
--- a/core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java
+++ b/core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.indices.query;
 
-import java.util.Map;
-
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryParser;
 
+import java.util.Map;
+
 public class IndicesQueriesRegistry extends AbstractComponent {
     private Map<String, QueryParser<?>> queryParsers;
 
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
index 24f87ee436f..8494939e46d 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
@@ -238,7 +238,7 @@ public class RecoveriesCollection {
                 return;
             }
             lastSeenAccessTime = accessTime;
-            logger.trace("[monitor] rescheduling check for [{}]. last access time is [{}]", lastSeenAccessTime);
+            logger.trace("[monitor] rescheduling check for [{}]. last access time is [{}]", recoveryId, lastSeenAccessTime);
             threadPool.schedule(checkInterval, ThreadPool.Names.GENERIC, this);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java
index 8d610dce05b..82595458479 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -32,31 +33,45 @@ import org.elasticsearch.common.unit.TimeValue;
 
 public class RecoverySettings extends AbstractComponent {
 
-    public static final Setting<ByteSizeValue> INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING = Setting.byteSizeSetting("indices.recovery.max_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB), true, Setting.Scope.CLUSTER);
+    public static final Setting<ByteSizeValue> INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING =
+        Setting.byteSizeSetting("indices.recovery.max_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB),
+            Property.Dynamic, Property.NodeScope);
 
     /**
      * how long to wait before retrying after issues cause by cluster state syncing between nodes
      * i.e., local node is not yet known on remote node, remote shard not yet started etc.
      */
-    public static final Setting<TimeValue> INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING = Setting.positiveTimeSetting("indices.recovery.retry_delay_state_sync", TimeValue.timeValueMillis(500), true, Setting.Scope.CLUSTER);
+    public static final Setting<TimeValue> INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING =
+        Setting.positiveTimeSetting("indices.recovery.retry_delay_state_sync", TimeValue.timeValueMillis(500),
+            Property.Dynamic, Property.NodeScope);
 
     /** how long to wait before retrying after network related issues */
-    public static final Setting<TimeValue> INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING = Setting.positiveTimeSetting("indices.recovery.retry_delay_network", TimeValue.timeValueSeconds(5), true, Setting.Scope.CLUSTER);
+    public static final Setting<TimeValue> INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING =
+        Setting.positiveTimeSetting("indices.recovery.retry_delay_network", TimeValue.timeValueSeconds(5),
+            Property.Dynamic, Property.NodeScope);
 
     /** timeout value to use for requests made as part of the recovery process */
-    public static final Setting<TimeValue> INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING = Setting.positiveTimeSetting("indices.recovery.internal_action_timeout", TimeValue.timeValueMinutes(15), true, Setting.Scope.CLUSTER);
+    public static final Setting<TimeValue> INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING =
+        Setting.positiveTimeSetting("indices.recovery.internal_action_timeout", TimeValue.timeValueMinutes(15),
+            Property.Dynamic, Property.NodeScope);
 
     /**
      * timeout value to use for requests made as part of the recovery process that are expected to take long time.
      * defaults to twice `indices.recovery.internal_action_timeout`.
      */
-    public static final Setting<TimeValue> INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING = Setting.timeSetting("indices.recovery.internal_action_long_timeout", (s) -> TimeValue.timeValueMillis(INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING.get(s).millis() * 2).toString(), TimeValue.timeValueSeconds(0), true,  Setting.Scope.CLUSTER);
+    public static final Setting<TimeValue> INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING =
+        Setting.timeSetting("indices.recovery.internal_action_long_timeout",
+            (s) -> TimeValue.timeValueMillis(INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING.get(s).millis() * 2).toString(),
+            TimeValue.timeValueSeconds(0), Property.Dynamic,  Property.NodeScope);
 
     /**
      * recoveries that don't show any activity for more then this interval will be failed.
      * defaults to `indices.recovery.internal_action_long_timeout`
      */
-    public static final Setting<TimeValue> INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING = Setting.timeSetting("indices.recovery.recovery_activity_timeout", (s) -> INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING.getRaw(s) , TimeValue.timeValueSeconds(0), true,  Setting.Scope.CLUSTER);
+    public static final Setting<TimeValue> INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING =
+        Setting.timeSetting("indices.recovery.recovery_activity_timeout",
+            (s) -> INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING.getRaw(s) , TimeValue.timeValueSeconds(0),
+            Property.Dynamic, Property.NodeScope);
 
     public static final ByteSizeValue DEFAULT_CHUNK_SIZE = new ByteSizeValue(512, ByteSizeUnit.KB);
 
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java
index 9a5c23fc2e1..aaf351f6056 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java
@@ -20,9 +20,9 @@
 package org.elasticsearch.indices.recovery;
 
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
@@ -83,7 +83,7 @@ public class RecoverySource extends AbstractComponent implements IndexEventListe
     }
 
     private RecoveryResponse recover(final StartRecoveryRequest request) throws IOException {
-        final IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex().getName());
+        final IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
         final IndexShard shard = indexService.getShard(request.shardId().id());
 
         // starting recovery from that our (the source) shard state is marking the shard to be in recovery mode as well, otherwise
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
index b92e2066af2..b609eb5d08a 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
@@ -137,7 +137,7 @@ public class RecoverySourceHandler {
                 }
             }
 
-            logger.trace("snapshot translog for recovery. current size is [{}]", translogView.totalOperations());
+            logger.trace("{} snapshot translog for recovery. current size is [{}]", shard.shardId(), translogView.totalOperations());
             try {
                 phase2(translogView.snapshot());
             } catch (Throwable e) {
@@ -289,7 +289,7 @@ public class RecoverySourceHandler {
                         RemoteTransportException exception = new RemoteTransportException("File corruption occurred on recovery but " +
                                 "checksums are ok", null);
                         exception.addSuppressed(targetException);
-                        logger.warn("{} Remote file corruption during finalization on node {}, recovering {}. local checksum OK",
+                        logger.warn("{} Remote file corruption during finalization of recovery on node {}. local checksum OK",
                                 corruptIndexException, shard.shardId(), request.targetNode());
                         throw exception;
                     } else {
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java
index dcbb0c7bedf..f0a0b13b872 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java
@@ -24,10 +24,10 @@ import org.apache.lucene.store.RateLimiter;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
@@ -90,8 +90,7 @@ public class RecoveryTargetService extends AbstractComponent implements IndexEve
 
     @Inject
     public RecoveryTargetService(Settings settings, ThreadPool threadPool, TransportService transportService, RecoverySettings
-            recoverySettings,
-                                 ClusterService clusterService) {
+            recoverySettings, ClusterService clusterService) {
         super(settings);
         this.threadPool = threadPool;
         this.transportService = transportService;
@@ -218,7 +217,7 @@ public class RecoveryTargetService extends AbstractComponent implements IndexEve
                         "operations")
                         .append(", took [").append(timeValueMillis(recoveryResponse.phase2Time)).append("]")
                         .append("\n");
-                logger.trace(sb.toString());
+                logger.trace("{}", sb);
             } else {
                 logger.debug("{} recovery done from [{}], took [{}]", request.shardId(), recoveryTarget.sourceNode(), recoveryTime);
             }
@@ -329,9 +328,13 @@ public class RecoveryTargetService extends AbstractComponent implements IndexEve
                         throw exception;
                     }
                     // in very rare cases a translog replay from primary is processed before a mapping update on this node
-                    // which causes local mapping changes. we want to wait until these mappings are processed.
+                    // which causes local mapping changes since the mapping (clusterstate) might not have arrived on this node.
+                    // we want to wait until these mappings are processed but also need to do some maintenance and roll back the
+                    // number of processed (completed) operations in this batch to ensure accounting is correct.
                     logger.trace("delaying recovery due to missing mapping changes (rolling back stats for [{}] ops)", exception, exception
                             .completedOperations());
+                    final RecoveryState.Translog translog = recoveryTarget.state().getTranslog();
+                    translog.decrementRecoveredOperations(exception.completedOperations()); // do the maintainance and rollback competed ops
                     // we do not need to use a timeout here since the entire recovery mechanism has an inactivity protection (it will be
                     // canceled)
                     observer.waitForNextChange(new ClusterStateObserver.Listener() {
diff --git a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
index d0aec817ee9..d2db41a7a0c 100644
--- a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
+++ b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
@@ -21,21 +21,23 @@ package org.elasticsearch.indices.store;
 
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
@@ -57,7 +59,6 @@ import org.elasticsearch.transport.TransportService;
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -69,7 +70,9 @@ import java.util.concurrent.atomic.AtomicInteger;
 public class IndicesStore extends AbstractComponent implements ClusterStateListener, Closeable {
 
     // TODO this class can be foled into either IndicesService and partially into IndicesClusterStateService there is no need for a separate public service
-    public static final Setting<TimeValue> INDICES_STORE_DELETE_SHARD_TIMEOUT = Setting.positiveTimeSetting("indices.store.delete.shard.timeout", new TimeValue(30, TimeUnit.SECONDS), false, Setting.Scope.CLUSTER);
+    public static final Setting<TimeValue> INDICES_STORE_DELETE_SHARD_TIMEOUT =
+        Setting.positiveTimeSetting("indices.store.delete.shard.timeout", new TimeValue(30, TimeUnit.SECONDS),
+            Property.NodeScope);
     public static final String ACTION_SHARD_EXISTS = "internal:index/shard/exists";
     private static final EnumSet<IndexShardState> ACTIVE_STATES = EnumSet.of(IndexShardState.STARTED, IndexShardState.RELOCATED);
     private final IndicesService indicesService;
@@ -113,7 +116,13 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
                 if (shardCanBeDeleted(event.state(), indexShardRoutingTable)) {
                     ShardId shardId = indexShardRoutingTable.shardId();
                     IndexService indexService = indicesService.indexService(indexRoutingTable.getIndex());
-                    IndexSettings indexSettings = indexService != null ? indexService.getIndexSettings() : new IndexSettings(event.state().getMetaData().index(indexRoutingTable.getIndex()), settings);
+                    final IndexSettings indexSettings;
+                    if (indexService == null) {
+                        IndexMetaData indexMetaData = event.state().getMetaData().getIndexSafe(indexRoutingTable.getIndex());
+                        indexSettings = new IndexSettings(indexMetaData, settings);
+                    } else {
+                        indexSettings = indexService.getIndexSettings();
+                    }
                     if (indicesService.canDeleteShardContent(shardId, indexSettings)) {
                         deleteShardIfExistElseWhere(event.state(), indexShardRoutingTable);
                     }
@@ -162,7 +171,7 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
 
     private void deleteShardIfExistElseWhere(ClusterState state, IndexShardRoutingTable indexShardRoutingTable) {
         List<Tuple<DiscoveryNode, ShardActiveRequest>> requests = new ArrayList<>(indexShardRoutingTable.size());
-        String indexUUID = state.getMetaData().index(indexShardRoutingTable.shardId().getIndex()).getIndexUUID();
+        String indexUUID = indexShardRoutingTable.shardId().getIndex().getUUID();
         ClusterName clusterName = state.getClusterName();
         for (ShardRouting shardRouting : indexShardRoutingTable) {
             // Node can't be null, because otherwise shardCanBeDeleted() would have returned false
@@ -348,7 +357,7 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
                 return null;
             }
             ShardId shardId = request.shardId;
-            IndexService indexService = indicesService.indexService(shardId.getIndexName());
+            IndexService indexService = indicesService.indexService(shardId.getIndex());
             if (indexService != null && indexService.indexUUID().equals(request.indexUUID)) {
                 return indexService.getShardOrNull(shardId.id());
             }
diff --git a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
index bcc2d7f74c4..35a34ebea1b 100644
--- a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
+++ b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
@@ -29,11 +29,11 @@ import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -126,7 +126,7 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
     @Override
     protected NodeStoreFilesMetaData nodeOperation(NodeRequest request) {
         if (request.unallocated) {
-            IndexService indexService = indicesService.indexService(request.shardId.getIndexName());
+            IndexService indexService = indicesService.indexService(request.shardId.getIndex());
             if (indexService == null) {
                 return new NodeStoreFilesMetaData(clusterService.localNode(), null);
             }
@@ -134,7 +134,7 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
                 return new NodeStoreFilesMetaData(clusterService.localNode(), null);
             }
         }
-        IndexMetaData metaData = clusterService.state().metaData().index(request.shardId.getIndexName());
+        IndexMetaData metaData = clusterService.state().metaData().index(request.shardId.getIndex());
         if (metaData == null) {
             return new NodeStoreFilesMetaData(clusterService.localNode(), null);
         }
@@ -150,7 +150,7 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
         long startTimeNS = System.nanoTime();
         boolean exists = false;
         try {
-            IndexService indexService = indicesService.indexService(shardId.getIndexName());
+            IndexService indexService = indicesService.indexService(shardId.getIndex());
             if (indexService != null) {
                 IndexShard indexShard = indexService.getShardOrNull(shardId.id());
                 if (indexShard != null) {
@@ -165,7 +165,7 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
                 }
             }
             // try and see if we an list unallocated
-            IndexMetaData metaData = clusterService.state().metaData().index(shardId.getIndexName());
+            IndexMetaData metaData = clusterService.state().metaData().index(shardId.getIndex());
             if (metaData == null) {
                 return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY);
             }
diff --git a/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java b/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java
index e4537b876fa..a30ae49ff7b 100644
--- a/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java
+++ b/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java
@@ -30,14 +30,15 @@ import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.bulk.TransportBulkAction;
 import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -67,7 +68,9 @@ import java.util.concurrent.locks.ReentrantLock;
  */
 public class IndicesTTLService extends AbstractLifecycleComponent<IndicesTTLService> {
 
-    public static final Setting<TimeValue> INDICES_TTL_INTERVAL_SETTING = Setting.positiveTimeSetting("indices.ttl.interval", TimeValue.timeValueSeconds(60), true, Setting.Scope.CLUSTER);
+    public static final Setting<TimeValue> INDICES_TTL_INTERVAL_SETTING =
+        Setting.positiveTimeSetting("indices.ttl.interval", TimeValue.timeValueSeconds(60),
+            Property.Dynamic, Property.NodeScope);
 
     private final ClusterService clusterService;
     private final IndicesService indicesService;
@@ -159,7 +162,7 @@ public class IndicesTTLService extends AbstractLifecycleComponent<IndicesTTLServ
             MetaData metaData = clusterService.state().metaData();
             for (IndexService indexService : indicesService) {
                 // check the value of disable_purge for this index
-                IndexMetaData indexMetaData = metaData.index(indexService.index().getName());
+                IndexMetaData indexMetaData = metaData.index(indexService.index());
                 if (indexMetaData == null) {
                     continue;
                 }
@@ -287,7 +290,7 @@ public class IndicesTTLService extends AbstractLifecycleComponent<IndicesTTLServ
                                 logger.error("bulk deletion failures for [{}]/[{}] items", failedItems, bulkResponse.getItems().length);
                             }
                         } else {
-                            logger.trace("bulk deletion took " + bulkResponse.getTookInMillis() + "ms");
+                            logger.trace("bulk deletion took {}ms", bulkResponse.getTookInMillis());
                         }
                     }
 
diff --git a/core/src/main/java/org/elasticsearch/ingest/IngestStats.java b/core/src/main/java/org/elasticsearch/ingest/IngestStats.java
new file mode 100644
index 00000000000..9ccc4126763
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/ingest/IngestStats.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+public class IngestStats implements Writeable<IngestStats>, ToXContent {
+    private final Stats totalStats;
+    private final Map<String, Stats> statsPerPipeline;
+
+    public IngestStats(StreamInput in) throws IOException {
+        this.totalStats = new Stats(in);
+        int size = in.readVInt();
+        this.statsPerPipeline = new HashMap<>(size);
+        for (int i = 0; i < size; i++) {
+            statsPerPipeline.put(in.readString(), new Stats(in));
+        }
+    }
+
+    public IngestStats(Stats totalStats, Map<String, Stats> statsPerPipeline) {
+        this.totalStats = totalStats;
+        this.statsPerPipeline = statsPerPipeline;
+    }
+
+    /**
+     * @return The accumulated stats for all pipelines
+     */
+    public Stats getTotalStats() {
+        return totalStats;
+    }
+
+    /**
+     * @return The stats on a per pipeline basis
+     */
+    public Map<String, Stats> getStatsPerPipeline() {
+        return statsPerPipeline;
+    }
+
+    @Override
+    public IngestStats readFrom(StreamInput in) throws IOException {
+        return new IngestStats(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        totalStats.writeTo(out);
+        out.writeVLong(statsPerPipeline.size());
+        for (Map.Entry<String, Stats> entry : statsPerPipeline.entrySet()) {
+            out.writeString(entry.getKey());
+            entry.getValue().writeTo(out);
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject("ingest");
+        builder.startObject("total");
+        totalStats.toXContent(builder, params);
+        builder.endObject();
+        builder.startObject("pipelines");
+        for (Map.Entry<String, Stats> entry : statsPerPipeline.entrySet()) {
+            builder.startObject(entry.getKey());
+            entry.getValue().toXContent(builder, params);
+            builder.endObject();
+        }
+        builder.endObject();
+        builder.endObject();
+        return builder;
+    }
+
+    public static class Stats implements Writeable<Stats>, ToXContent {
+
+        private final long ingestCount;
+        private final long ingestTimeInMillis;
+        private final long ingestCurrent;
+        private final long ingestFailedCount;
+
+        public Stats(StreamInput in) throws IOException {
+            ingestCount = in.readVLong();
+            ingestTimeInMillis = in.readVLong();
+            ingestCurrent = in.readVLong();
+            ingestFailedCount = in.readVLong();
+        }
+
+        public Stats(long ingestCount, long ingestTimeInMillis, long ingestCurrent, long ingestFailedCount) {
+            this.ingestCount = ingestCount;
+            this.ingestTimeInMillis = ingestTimeInMillis;
+            this.ingestCurrent = ingestCurrent;
+            this.ingestFailedCount = ingestFailedCount;
+        }
+
+        /**
+         * @return The total number of executed ingest preprocessing operations.
+         */
+        public long getIngestCount() {
+            return ingestCount;
+        }
+
+        /**
+         *
+         * @return The total time spent of ingest preprocessing in millis.
+         */
+        public long getIngestTimeInMillis() {
+            return ingestTimeInMillis;
+        }
+
+        /**
+         * @return The total number of ingest preprocessing operations currently executing.
+         */
+        public long getIngestCurrent() {
+            return ingestCurrent;
+        }
+
+        /**
+         * @return The total number of ingest preprocessing operations that have failed.
+         */
+        public long getIngestFailedCount() {
+            return ingestFailedCount;
+        }
+
+        @Override
+        public Stats readFrom(StreamInput in) throws IOException {
+            return new Stats(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeVLong(ingestCount);
+            out.writeVLong(ingestTimeInMillis);
+            out.writeVLong(ingestCurrent);
+            out.writeVLong(ingestFailedCount);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.field("count", ingestCount);
+            builder.timeValueField("time_in_millis", "time", ingestTimeInMillis, TimeUnit.MILLISECONDS);
+            builder.field("current", ingestCurrent);
+            builder.field("failed", ingestFailedCount);
+            return builder;
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java b/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
index 3f0de550782..94c79db30a0 100644
--- a/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
+++ b/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
@@ -19,23 +19,36 @@
 
 package org.elasticsearch.ingest;
 
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.metrics.CounterMetric;
+import org.elasticsearch.common.metrics.MeanMetric;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.ingest.core.Pipeline;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 
-public class PipelineExecutionService {
+public class PipelineExecutionService implements ClusterStateListener {
 
     private final PipelineStore store;
     private final ThreadPool threadPool;
 
+    private final StatsHolder totalStats = new StatsHolder();
+    private volatile Map<String, StatsHolder> statsHolderPerPipeline = Collections.emptyMap();
+
     public PipelineExecutionService(PipelineStore store, ThreadPool threadPool) {
         this.store = store;
         this.threadPool = threadPool;
@@ -89,29 +102,85 @@ public class PipelineExecutionService {
         });
     }
 
-    private void innerExecute(IndexRequest indexRequest, Pipeline pipeline) throws Exception {
-        String index = indexRequest.index();
-        String type = indexRequest.type();
-        String id = indexRequest.id();
-        String routing = indexRequest.routing();
-        String parent = indexRequest.parent();
-        String timestamp = indexRequest.timestamp();
-        String ttl = indexRequest.ttl() == null ? null : indexRequest.ttl().toString();
-        Map<String, Object> sourceAsMap = indexRequest.sourceAsMap();
-        IngestDocument ingestDocument = new IngestDocument(index, type, id, routing, parent, timestamp, ttl, sourceAsMap);
-        pipeline.execute(ingestDocument);
+    public IngestStats stats() {
+        Map<String, StatsHolder> statsHolderPerPipeline = this.statsHolderPerPipeline;
 
-        Map<IngestDocument.MetaData, String> metadataMap = ingestDocument.extractMetadata();
-        //it's fine to set all metadata fields all the time, as ingest document holds their starting values
-        //before ingestion, which might also get modified during ingestion.
-        indexRequest.index(metadataMap.get(IngestDocument.MetaData.INDEX));
-        indexRequest.type(metadataMap.get(IngestDocument.MetaData.TYPE));
-        indexRequest.id(metadataMap.get(IngestDocument.MetaData.ID));
-        indexRequest.routing(metadataMap.get(IngestDocument.MetaData.ROUTING));
-        indexRequest.parent(metadataMap.get(IngestDocument.MetaData.PARENT));
-        indexRequest.timestamp(metadataMap.get(IngestDocument.MetaData.TIMESTAMP));
-        indexRequest.ttl(metadataMap.get(IngestDocument.MetaData.TTL));
-        indexRequest.source(ingestDocument.getSourceAndMetadata());
+        Map<String, IngestStats.Stats> statsPerPipeline = new HashMap<>(statsHolderPerPipeline.size());
+        for (Map.Entry<String, StatsHolder> entry : statsHolderPerPipeline.entrySet()) {
+            statsPerPipeline.put(entry.getKey(), entry.getValue().createStats());
+        }
+
+        return new IngestStats(totalStats.createStats(), statsPerPipeline);
+    }
+
+    @Override
+    public void clusterChanged(ClusterChangedEvent event) {
+        IngestMetadata ingestMetadata = event.state().getMetaData().custom(IngestMetadata.TYPE);
+        if (ingestMetadata != null) {
+            updatePipelineStats(ingestMetadata);
+        }
+    }
+
+    void updatePipelineStats(IngestMetadata ingestMetadata) {
+        boolean changed = false;
+        Map<String, StatsHolder> newStatsPerPipeline = new HashMap<>(statsHolderPerPipeline);
+        for (String pipeline : newStatsPerPipeline.keySet()) {
+            if (ingestMetadata.getPipelines().containsKey(pipeline) == false) {
+                newStatsPerPipeline.remove(pipeline);
+                changed = true;
+            }
+        }
+        for (String pipeline : ingestMetadata.getPipelines().keySet()) {
+            if (newStatsPerPipeline.containsKey(pipeline) == false) {
+                newStatsPerPipeline.put(pipeline, new StatsHolder());
+                changed = true;
+            }
+        }
+
+        if (changed) {
+            statsHolderPerPipeline = Collections.unmodifiableMap(newStatsPerPipeline);
+        }
+    }
+
+    private void innerExecute(IndexRequest indexRequest, Pipeline pipeline) throws Exception {
+        long startTimeInNanos = System.nanoTime();
+        // the pipeline specific stat holder may not exist and that is fine:
+        // (e.g. the pipeline may have been removed while we're ingesting a document
+        Optional<StatsHolder> pipelineStats = Optional.ofNullable(statsHolderPerPipeline.get(pipeline.getId()));
+        try {
+            totalStats.preIngest();
+            pipelineStats.ifPresent(StatsHolder::preIngest);
+            String index = indexRequest.index();
+            String type = indexRequest.type();
+            String id = indexRequest.id();
+            String routing = indexRequest.routing();
+            String parent = indexRequest.parent();
+            String timestamp = indexRequest.timestamp();
+            String ttl = indexRequest.ttl() == null ? null : indexRequest.ttl().toString();
+            Map<String, Object> sourceAsMap = indexRequest.sourceAsMap();
+            IngestDocument ingestDocument = new IngestDocument(index, type, id, routing, parent, timestamp, ttl, sourceAsMap);
+            pipeline.execute(ingestDocument);
+
+            Map<IngestDocument.MetaData, String> metadataMap = ingestDocument.extractMetadata();
+            //it's fine to set all metadata fields all the time, as ingest document holds their starting values
+            //before ingestion, which might also get modified during ingestion.
+            indexRequest.index(metadataMap.get(IngestDocument.MetaData.INDEX));
+            indexRequest.type(metadataMap.get(IngestDocument.MetaData.TYPE));
+            indexRequest.id(metadataMap.get(IngestDocument.MetaData.ID));
+            indexRequest.routing(metadataMap.get(IngestDocument.MetaData.ROUTING));
+            indexRequest.parent(metadataMap.get(IngestDocument.MetaData.PARENT));
+            indexRequest.timestamp(metadataMap.get(IngestDocument.MetaData.TIMESTAMP));
+            indexRequest.ttl(metadataMap.get(IngestDocument.MetaData.TTL));
+            indexRequest.source(ingestDocument.getSourceAndMetadata());
+        } catch (Exception e) {
+            totalStats.ingestFailed();
+            pipelineStats.ifPresent(StatsHolder::ingestFailed);
+            throw e;
+        } finally {
+            long ingestTimeInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeInNanos);
+            totalStats.postIngest(ingestTimeInMillis);
+            pipelineStats.ifPresent(statsHolder -> statsHolder.postIngest(ingestTimeInMillis));
+        }
     }
 
     private Pipeline getPipeline(String pipelineId) {
@@ -121,4 +190,30 @@ public class PipelineExecutionService {
         }
         return pipeline;
     }
+
+    static class StatsHolder {
+
+        private final MeanMetric ingestMetric = new MeanMetric();
+        private final CounterMetric ingestCurrent = new CounterMetric();
+        private final CounterMetric ingestFailed = new CounterMetric();
+
+        void preIngest() {
+            ingestCurrent.inc();
+        }
+
+        void postIngest(long ingestTimeInMillis) {
+            ingestCurrent.dec();
+            ingestMetric.inc(ingestTimeInMillis);
+        }
+
+        void ingestFailed() {
+            ingestFailed.inc();
+        }
+
+        IngestStats.Stats createStats() {
+            return new IngestStats.Stats(ingestMetric.count(), ingestMetric.sum(), ingestCurrent.count(), ingestFailed.count());
+        }
+
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
index ac2df419f55..7e0dc1b4ffa 100644
--- a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
+++ b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
@@ -28,11 +28,11 @@ import org.elasticsearch.action.ingest.PutPipelineRequest;
 import org.elasticsearch.action.ingest.WritePipelineResponse;
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
@@ -40,7 +40,6 @@ import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.ingest.core.IngestInfo;
 import org.elasticsearch.ingest.core.Pipeline;
 import org.elasticsearch.ingest.core.Processor;
-import org.elasticsearch.ingest.core.ProcessorInfo;
 import org.elasticsearch.ingest.core.TemplateService;
 import org.elasticsearch.script.ScriptService;
 
@@ -51,7 +50,6 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 
 public class PipelineStore extends AbstractComponent implements Closeable, ClusterStateListener {
 
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java b/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java
index ddf3781d1a6..16b3aa10a22 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java
+++ b/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java
@@ -28,15 +28,16 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 /**
  * A Processor that executes a list of other "processors". It executes a separate list of
  * "onFailureProcessors" when any of the processors throw an {@link Exception}.
  */
 public class CompoundProcessor implements Processor {
-    static final String ON_FAILURE_MESSAGE_FIELD = "on_failure_message";
-    static final String ON_FAILURE_PROCESSOR_TYPE_FIELD = "on_failure_processor_type";
-    static final String ON_FAILURE_PROCESSOR_TAG_FIELD = "on_failure_processor_tag";
+    public static final String ON_FAILURE_MESSAGE_FIELD = "on_failure_message";
+    public static final String ON_FAILURE_PROCESSOR_TYPE_FIELD = "on_failure_processor_type";
+    public static final String ON_FAILURE_PROCESSOR_TAG_FIELD = "on_failure_processor_tag";
 
     private final List<Processor> processors;
     private final List<Processor> onFailureProcessors;
@@ -84,7 +85,7 @@ public class CompoundProcessor implements Processor {
 
     @Override
     public String getTag() {
-        return "compound-processor-" + Objects.hash(processors, onFailureProcessors);
+        return "CompoundProcessor-" + flattenProcessors().stream().map(Processor::getTag).collect(Collectors.joining("-"));
     }
 
     @Override
@@ -104,18 +105,27 @@ public class CompoundProcessor implements Processor {
     }
 
     void executeOnFailure(IngestDocument ingestDocument, Exception cause, String failedProcessorType, String failedProcessorTag) throws Exception {
-        Map<String, String> ingestMetadata = ingestDocument.getIngestMetadata();
         try {
-            ingestMetadata.put(ON_FAILURE_MESSAGE_FIELD, cause.getMessage());
-            ingestMetadata.put(ON_FAILURE_PROCESSOR_TYPE_FIELD, failedProcessorType);
-            ingestMetadata.put(ON_FAILURE_PROCESSOR_TAG_FIELD, failedProcessorTag);
+            putFailureMetadata(ingestDocument, cause, failedProcessorType, failedProcessorTag);
             for (Processor processor : onFailureProcessors) {
                 processor.execute(ingestDocument);
             }
         } finally {
-            ingestMetadata.remove(ON_FAILURE_MESSAGE_FIELD);
-            ingestMetadata.remove(ON_FAILURE_PROCESSOR_TYPE_FIELD);
-            ingestMetadata.remove(ON_FAILURE_PROCESSOR_TAG_FIELD);
+            removeFailureMetadata(ingestDocument);
         }
     }
+
+    private void putFailureMetadata(IngestDocument ingestDocument, Exception cause, String failedProcessorType, String failedProcessorTag) {
+        Map<String, String> ingestMetadata = ingestDocument.getIngestMetadata();
+        ingestMetadata.put(ON_FAILURE_MESSAGE_FIELD, cause.getMessage());
+        ingestMetadata.put(ON_FAILURE_PROCESSOR_TYPE_FIELD, failedProcessorType);
+        ingestMetadata.put(ON_FAILURE_PROCESSOR_TAG_FIELD, failedProcessorTag);
+    }
+
+    private void removeFailureMetadata(IngestDocument ingestDocument) {
+        Map<String, String> ingestMetadata = ingestDocument.getIngestMetadata();
+        ingestMetadata.remove(ON_FAILURE_MESSAGE_FIELD);
+        ingestMetadata.remove(ON_FAILURE_PROCESSOR_TYPE_FIELD);
+        ingestMetadata.remove(ON_FAILURE_PROCESSOR_TAG_FIELD);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/IngestInfo.java b/core/src/main/java/org/elasticsearch/ingest/core/IngestInfo.java
index 8625e1d8884..d128732203f 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/IngestInfo.java
+++ b/core/src/main/java/org/elasticsearch/ingest/core/IngestInfo.java
@@ -22,6 +22,7 @@ package org.elasticsearch.ingest.core;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
@@ -32,17 +33,22 @@ import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
+import java.util.TreeSet;
 
-public class IngestInfo implements Streamable, ToXContent {
+public class IngestInfo implements Writeable<IngestInfo>, ToXContent {
 
-    private Set<ProcessorInfo> processors;
+    private final Set<ProcessorInfo> processors;
 
-    public IngestInfo() {
-        processors = Collections.emptySet();
+    public IngestInfo(StreamInput in) throws IOException {
+        this(Collections.emptyList());
+        final int size = in.readVInt();
+        for (int i = 0; i < size; i++) {
+            processors.add(new ProcessorInfo(in));
+        }
     }
 
     public IngestInfo(List<ProcessorInfo> processors) {
-        this.processors = new LinkedHashSet<>(processors);
+        this.processors = new TreeSet<>(processors);  // we use a treeset here to have a test-able / predictable order
     }
 
     public Iterable<ProcessorInfo> getProcessors() {
@@ -54,15 +60,8 @@ public class IngestInfo implements Streamable, ToXContent {
     }
 
     @Override
-    public void readFrom(StreamInput in) throws IOException {
-        int size = in.readVInt();
-        Set<ProcessorInfo> processors = new LinkedHashSet<>(size);
-        for (int i = 0; i < size; i++) {
-            ProcessorInfo info = new ProcessorInfo();
-            info.readFrom(in);
-            processors.add(info);
-        }
-        this.processors = processors;
+    public IngestInfo readFrom(StreamInput in) throws IOException {
+        return new IngestInfo(in);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/Pipeline.java b/core/src/main/java/org/elasticsearch/ingest/core/Pipeline.java
index 821a44c0a96..aaae929e0a9 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/Pipeline.java
+++ b/core/src/main/java/org/elasticsearch/ingest/core/Pipeline.java
@@ -68,6 +68,13 @@ public final class Pipeline {
         return description;
     }
 
+    /**
+     * Get the underlying {@link CompoundProcessor} containing the Pipeline's processors
+     */
+    public CompoundProcessor getCompoundProcessor() {
+        return compoundProcessor;
+    }
+
     /**
      * Unmodifiable list containing each processor that operates on the data.
      */
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/ProcessorInfo.java b/core/src/main/java/org/elasticsearch/ingest/core/ProcessorInfo.java
index 64c3d19719b..f652b182919 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/ProcessorInfo.java
+++ b/core/src/main/java/org/elasticsearch/ingest/core/ProcessorInfo.java
@@ -22,16 +22,18 @@ package org.elasticsearch.ingest.core;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
 
-public class ProcessorInfo implements Streamable, ToXContent {
+public class ProcessorInfo implements Writeable<ProcessorInfo>, ToXContent, Comparable<ProcessorInfo> {
 
-    private String type;
+    private final String type;
 
-    ProcessorInfo() {
+    public ProcessorInfo(StreamInput input) throws IOException {
+        type = input.readString();
     }
 
     public ProcessorInfo(String type) {
@@ -46,8 +48,8 @@ public class ProcessorInfo implements Streamable, ToXContent {
     }
 
     @Override
-    public void readFrom(StreamInput in) throws IOException {
-        this.type = in.readString();
+    public ProcessorInfo readFrom(StreamInput in) throws IOException {
+        return new ProcessorInfo(in);
     }
 
     @Override
@@ -78,4 +80,9 @@ public class ProcessorInfo implements Streamable, ToXContent {
     public int hashCode() {
         return type.hashCode();
     }
+
+    @Override
+    public int compareTo(ProcessorInfo o) {
+        return type.compareTo(o.type);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/ForEachProcessor.java b/core/src/main/java/org/elasticsearch/ingest/processor/ForEachProcessor.java
index 5b101fbfb32..b6f6a85d219 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/ForEachProcessor.java
+++ b/core/src/main/java/org/elasticsearch/ingest/processor/ForEachProcessor.java
@@ -59,11 +59,8 @@ public final class ForEachProcessor extends AbstractProcessor {
         List<Object> values = ingestDocument.getFieldValue(field, List.class);
         List<Object> newValues = new ArrayList<>(values.size());
         for (Object value : values) {
-            Map<String, Object> innerSource = new HashMap<>();
-            innerSource.put("_value", value);
-            for (IngestDocument.MetaData metaData : IngestDocument.MetaData.values()) {
-                innerSource.put(metaData.getFieldName(), ingestDocument.getSourceAndMetadata().get(metaData.getFieldName()));
-            }
+            Map<String, Object> innerSource = new HashMap<>(ingestDocument.getSourceAndMetadata());
+            innerSource.put("_value", value); // scalar value to access the list item being evaluated
             IngestDocument innerIngestDocument = new IngestDocument(innerSource, ingestDocument.getIngestMetadata());
             for (Processor processor : processors) {
                 processor.execute(innerIngestDocument);
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/TrackingResultProcessor.java b/core/src/main/java/org/elasticsearch/ingest/processor/TrackingResultProcessor.java
new file mode 100644
index 00000000000..af820318d83
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/ingest/processor/TrackingResultProcessor.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.processor;
+
+import org.elasticsearch.action.ingest.SimulateProcessorResult;
+import org.elasticsearch.ingest.core.CompoundProcessor;
+import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.core.Processor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Processor to be used within Simulate API to keep track of processors executed in pipeline.
+ */
+public final class TrackingResultProcessor implements Processor {
+
+    private final Processor actualProcessor;
+    private final List<SimulateProcessorResult> processorResultList;
+
+    public TrackingResultProcessor(Processor actualProcessor, List<SimulateProcessorResult> processorResultList) {
+        this.processorResultList = processorResultList;
+        if (actualProcessor instanceof CompoundProcessor) {
+            CompoundProcessor trackedCompoundProcessor = decorate((CompoundProcessor) actualProcessor, processorResultList);
+            this.actualProcessor = trackedCompoundProcessor;
+        } else {
+            this.actualProcessor = actualProcessor;
+        }
+    }
+
+    @Override
+    public void execute(IngestDocument ingestDocument) throws Exception {
+        try {
+            actualProcessor.execute(ingestDocument);
+            processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(), new IngestDocument(ingestDocument)));
+        } catch (Exception e) {
+            processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(), e));
+            throw e;
+        }
+    }
+
+    @Override
+    public String getType() {
+        return actualProcessor.getType();
+    }
+
+    @Override
+    public String getTag() {
+        return actualProcessor.getTag();
+    }
+
+    public static CompoundProcessor decorate(CompoundProcessor compoundProcessor, List<SimulateProcessorResult> processorResultList) {
+        List<Processor> processors = new ArrayList<>(compoundProcessor.getProcessors().size());
+        for (Processor processor : compoundProcessor.getProcessors()) {
+            if (processor instanceof CompoundProcessor) {
+                processors.add(decorate((CompoundProcessor) processor, processorResultList));
+            } else {
+                processors.add(new TrackingResultProcessor(processor, processorResultList));
+            }
+        }
+        List<Processor> onFailureProcessors = new ArrayList<>(compoundProcessor.getProcessors().size());
+        for (Processor processor : compoundProcessor.getOnFailureProcessors()) {
+            if (processor instanceof CompoundProcessor) {
+                onFailureProcessors.add(decorate((CompoundProcessor) processor, processorResultList));
+            } else {
+                onFailureProcessors.add(new TrackingResultProcessor(processor, processorResultList));
+            }
+        }
+        return new CompoundProcessor(processors, onFailureProcessors);
+    }
+}
+
diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java
index 99a78f13a07..0287d5c522c 100644
--- a/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java
@@ -21,6 +21,7 @@ package org.elasticsearch.monitor.fs;
 
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.SingleObjectCache;
@@ -37,7 +38,8 @@ public class FsService extends AbstractComponent {
     private final SingleObjectCache<FsInfo> fsStatsCache;
 
     public final static Setting<TimeValue> REFRESH_INTERVAL_SETTING =
-        Setting.timeSetting("monitor.fs.refresh_interval", TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(1), false, Setting.Scope.CLUSTER);
+        Setting.timeSetting("monitor.fs.refresh_interval", TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(1),
+            Property.NodeScope);
 
     public FsService(Settings settings, NodeEnvironment nodeEnvironment) throws IOException {
         super(settings);
diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
index 97c813a0fe3..5a2d591c7dc 100644
--- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
@@ -21,7 +21,7 @@ package org.elasticsearch.monitor.jvm;
 
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Scope;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
@@ -47,12 +47,14 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
 
     private volatile ScheduledFuture scheduledFuture;
 
-    public final static Setting<Boolean> ENABLED_SETTING = Setting.boolSetting("monitor.jvm.gc.enabled", true, false, Scope.CLUSTER);
+    public final static Setting<Boolean> ENABLED_SETTING =
+        Setting.boolSetting("monitor.jvm.gc.enabled", true, Property.NodeScope);
     public final static Setting<TimeValue> REFRESH_INTERVAL_SETTING =
-        Setting.timeSetting("monitor.jvm.gc.refresh_interval", TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(1), false, Scope.CLUSTER);
+        Setting.timeSetting("monitor.jvm.gc.refresh_interval", TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(1),
+            Property.NodeScope);
 
     private static String GC_COLLECTOR_PREFIX = "monitor.jvm.gc.collector.";
-    public final static Setting<Settings> GC_SETTING = Setting.groupSetting(GC_COLLECTOR_PREFIX, false, Scope.CLUSTER);
+    public final static Setting<Settings> GC_SETTING = Setting.groupSetting(GC_COLLECTOR_PREFIX, Property.NodeScope);
 
     static class GcThreshold {
         public final String name;
diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmService.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmService.java
index fbec6cda168..e91c05e75ac 100644
--- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmService.java
@@ -21,6 +21,7 @@ package org.elasticsearch.monitor.jvm;
 
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 
@@ -36,7 +37,8 @@ public class JvmService extends AbstractComponent {
     private JvmStats jvmStats;
 
     public final static Setting<TimeValue> REFRESH_INTERVAL_SETTING =
-        Setting.timeSetting("monitor.jvm.refresh_interval", TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(1), false, Setting.Scope.CLUSTER);
+        Setting.timeSetting("monitor.jvm.refresh_interval", TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(1),
+            Property.NodeScope);
 
     public JvmService(Settings settings) {
         super(settings);
diff --git a/core/src/main/java/org/elasticsearch/monitor/os/OsService.java b/core/src/main/java/org/elasticsearch/monitor/os/OsService.java
index 5f836c6f928..d452094d7b0 100644
--- a/core/src/main/java/org/elasticsearch/monitor/os/OsService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/os/OsService.java
@@ -21,6 +21,7 @@ package org.elasticsearch.monitor.os;
 
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.SingleObjectCache;
@@ -38,7 +39,8 @@ public class OsService extends AbstractComponent {
     private SingleObjectCache<OsStats> osStatsCache;
 
     public final static Setting<TimeValue> REFRESH_INTERVAL_SETTING =
-        Setting.timeSetting("monitor.os.refresh_interval", TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(1), false, Setting.Scope.CLUSTER);
+        Setting.timeSetting("monitor.os.refresh_interval", TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(1),
+            Property.NodeScope);
 
     public OsService(Settings settings) {
         super(settings);
diff --git a/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java b/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java
index 9e3283af4fc..30c24f34c66 100644
--- a/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java
@@ -21,6 +21,7 @@ package org.elasticsearch.monitor.process;
 
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.SingleObjectCache;
@@ -35,7 +36,8 @@ public final class ProcessService extends AbstractComponent {
     private final SingleObjectCache<ProcessStats> processStatsCache;
 
     public final static Setting<TimeValue> REFRESH_INTERVAL_SETTING =
-        Setting.timeSetting("monitor.process.refresh_interval", TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(1), false, Setting.Scope.CLUSTER);
+        Setting.timeSetting("monitor.process.refresh_interval", TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(1),
+            Property.NodeScope);
 
     public ProcessService(Settings settings) {
         super(settings);
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index e279d3e819f..6e270ffc3ff 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -30,14 +30,15 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.client.node.NodeClientModule;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterNameModule;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.MasterNodeChangePredicate;
+import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.cluster.routing.RoutingService;
-import org.elasticsearch.cluster.service.InternalClusterService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.StopWatch;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleComponent;
@@ -53,6 +54,7 @@ import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.transport.BoundTransportAddress;
@@ -81,8 +83,6 @@ import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.node.service.NodeService;
-import org.elasticsearch.percolator.PercolatorModule;
-import org.elasticsearch.percolator.PercolatorService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsModule;
 import org.elasticsearch.plugins.PluginsService;
@@ -129,17 +129,23 @@ import static org.elasticsearch.common.settings.Settings.settingsBuilder;
  */
 public class Node implements Closeable {
 
-    public static final Setting<Boolean> WRITE_PORTS_FIELD_SETTING = Setting.boolSetting("node.portsfile", false, false, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> NODE_CLIENT_SETTING = Setting.boolSetting("node.client", false, false, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> NODE_DATA_SETTING = Setting.boolSetting("node.data", true, false, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> NODE_MASTER_SETTING = Setting.boolSetting("node.master", true, false, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> NODE_LOCAL_SETTING = Setting.boolSetting("node.local", false, false, Setting.Scope.CLUSTER);
-    public static final Setting<String> NODE_MODE_SETTING = new Setting<>("node.mode", "network", Function.identity(), false, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> NODE_INGEST_SETTING = Setting.boolSetting("node.ingest", true, false, Setting.Scope.CLUSTER);
-    public static final Setting<String> NODE_NAME_SETTING = Setting.simpleString("node.name", false, Setting.Scope.CLUSTER);
+    public static final Setting<Boolean> WRITE_PORTS_FIELD_SETTING =
+        Setting.boolSetting("node.portsfile", false, Property.NodeScope);
+    public static final Setting<Boolean> NODE_CLIENT_SETTING =
+        Setting.boolSetting("node.client", false, Property.NodeScope);
+    public static final Setting<Boolean> NODE_DATA_SETTING = Setting.boolSetting("node.data", true, Property.NodeScope);
+    public static final Setting<Boolean> NODE_MASTER_SETTING =
+        Setting.boolSetting("node.master", true, Property.NodeScope);
+    public static final Setting<Boolean> NODE_LOCAL_SETTING =
+        Setting.boolSetting("node.local", false, Property.NodeScope);
+    public static final Setting<String> NODE_MODE_SETTING =
+        new Setting<>("node.mode", "network", Function.identity(), Property.NodeScope);
+    public static final Setting<Boolean> NODE_INGEST_SETTING =
+        Setting.boolSetting("node.ingest", true, Property.NodeScope);
+    public static final Setting<String> NODE_NAME_SETTING = Setting.simpleString("node.name", Property.NodeScope);
     // this sucks that folks can mistype client etc and get away with it.
     // TODO: we should move this to node.attribute.${name} = ${value} instead.
-    public static final Setting<Settings> NODE_ATTRIBUTES = Setting.groupSetting("node.", false, Setting.Scope.CLUSTER);
+    public static final Setting<Settings> NODE_ATTRIBUTES = Setting.groupSetting("node.", Property.NodeScope);
 
 
     private static final String CLIENT_TYPE = "node";
@@ -217,7 +223,6 @@ public class Node implements Closeable {
             modules.add(new ActionModule(DiscoveryNode.ingestNode(settings), false));
             modules.add(new GatewayModule(settings));
             modules.add(new NodeClientModule());
-            modules.add(new PercolatorModule());
             modules.add(new ResourceWatcherModule());
             modules.add(new RepositoriesModule());
             modules.add(new TribeModule());
@@ -290,9 +295,11 @@ public class Node implements Closeable {
         injector.getInstance(MonitorService.class).start();
         injector.getInstance(RestController.class).start();
 
-        assert injector.getInstance(ClusterService.class) instanceof InternalClusterService :
-                "node cluster service implementation must inherit from InternalClusterService";
-        final InternalClusterService clusterService = (InternalClusterService) injector.getInstance(ClusterService.class);
+        final ClusterService clusterService = injector.getInstance(ClusterService.class);
+
+        final NodeConnectionsService nodeConnectionsService = injector.getInstance(NodeConnectionsService.class);
+        nodeConnectionsService.start();
+        clusterService.setNodeConnectionsService(nodeConnectionsService);
 
         // TODO hack around circular dependencies problems
         injector.getInstance(GatewayAllocator.class).setReallocation(clusterService, injector.getInstance(RoutingService.class));
@@ -311,6 +318,15 @@ public class Node implements Closeable {
         // Start the transport service now so the publish address will be added to the local disco node in ClusterService
         TransportService transportService = injector.getInstance(TransportService.class);
         transportService.start();
+        DiscoveryNode localNode = injector.getInstance(DiscoveryNodeService.class)
+                .buildLocalNode(transportService.boundAddress().publishAddress());
+
+        // TODO: need to find a cleaner way to start/construct a service with some initial parameters,
+        // playing nice with the life cycle interfaces
+        clusterService.setLocalNode(localNode);
+        transportService.setLocalNode(localNode);
+        clusterService.add(transportService.getTaskManager());
+
         clusterService.start();
 
         // start after cluster service so the local disco is known
@@ -392,6 +408,7 @@ public class Node implements Closeable {
         injector.getInstance(RoutingService.class).stop();
         injector.getInstance(ClusterService.class).stop();
         injector.getInstance(Discovery.class).stop();
+        injector.getInstance(NodeConnectionsService.class).stop();
         injector.getInstance(MonitorService.class).stop();
         injector.getInstance(GatewayService.class).stop();
         injector.getInstance(SearchService.class).stop();
@@ -449,6 +466,8 @@ public class Node implements Closeable {
         toClose.add(injector.getInstance(RoutingService.class));
         toClose.add(() -> stopWatch.stop().start("cluster"));
         toClose.add(injector.getInstance(ClusterService.class));
+        toClose.add(() -> stopWatch.stop().start("node_connections_service"));
+        toClose.add(injector.getInstance(NodeConnectionsService.class));
         toClose.add(() -> stopWatch.stop().start("discovery"));
         toClose.add(injector.getInstance(Discovery.class));
         toClose.add(() -> stopWatch.stop().start("monitor"));
@@ -461,8 +480,6 @@ public class Node implements Closeable {
         toClose.add(injector.getInstance(RestController.class));
         toClose.add(() -> stopWatch.stop().start("transport"));
         toClose.add(injector.getInstance(TransportService.class));
-        toClose.add(() -> stopWatch.stop().start("percolator_service"));
-        toClose.add(injector.getInstance(PercolatorService.class));
 
         for (Class<? extends LifecycleComponent> plugin : pluginsService.nodeServices()) {
             toClose.add(() -> stopWatch.stop().start("plugin(" + plugin.getName() + ")"));
diff --git a/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java b/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
index faf449586c1..8864a70ccdc 100644
--- a/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
+++ b/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
@@ -23,9 +23,10 @@ import org.elasticsearch.bootstrap.BootstrapInfo;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.cli.Terminal;
+import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.env.Environment;
@@ -52,12 +53,13 @@ import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 public class InternalSettingsPreparer {
 
     private static final String[] ALLOWED_SUFFIXES = {".yml", ".yaml", ".json", ".properties"};
-    static final String[] PROPERTY_PREFIXES = {"es.", "elasticsearch."};
-    static final String[] PROPERTY_DEFAULTS_PREFIXES = {"es.default.", "elasticsearch.default."};
+    static final String PROPERTY_PREFIX = "es.";
+    static final String PROPERTY_DEFAULTS_PREFIX = "es.default.";
 
     public static final String SECRET_PROMPT_VALUE = "${prompt.secret}";
     public static final String TEXT_PROMPT_VALUE = "${prompt.text}";
-    public static final Setting<Boolean> IGNORE_SYSTEM_PROPERTIES_SETTING = Setting.boolSetting("config.ignore_system_properties", false, false, Setting.Scope.CLUSTER);
+    public static final Setting<Boolean> IGNORE_SYSTEM_PROPERTIES_SETTING =
+        Setting.boolSetting("config.ignore_system_properties", false, Property.NodeScope);
 
     /**
      * Prepares the settings by gathering all elasticsearch system properties and setting defaults.
@@ -124,13 +126,9 @@ public class InternalSettingsPreparer {
         output.put(input);
         if (useSystemProperties(input)) {
             if (loadDefaults) {
-                for (String prefix : PROPERTY_DEFAULTS_PREFIXES) {
-                    output.putProperties(prefix, BootstrapInfo.getSystemProperties());
-                }
-            }
-            for (String prefix : PROPERTY_PREFIXES) {
-                output.putProperties(prefix, BootstrapInfo.getSystemProperties(), PROPERTY_DEFAULTS_PREFIXES);
+                output.putProperties(PROPERTY_DEFAULTS_PREFIX, BootstrapInfo.getSystemProperties());
             }
+            output.putProperties(PROPERTY_PREFIX, BootstrapInfo.getSystemProperties(), PROPERTY_DEFAULTS_PREFIX);
         }
         output.replacePropertyPlaceholders();
     }
diff --git a/core/src/main/java/org/elasticsearch/node/service/NodeService.java b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
index 88b2fe48868..cb11fc02443 100644
--- a/core/src/main/java/org/elasticsearch/node/service/NodeService.java
+++ b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
@@ -24,7 +24,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
-import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
@@ -90,6 +90,7 @@ public class NodeService extends AbstractComponent implements Closeable {
         this.ingestService = new IngestService(settings, threadPool, processorsRegistryBuilder);
         this.settingsFilter = settingsFilter;
         clusterService.add(ingestService.getPipelineStore());
+        clusterService.add(ingestService.getPipelineExecutionService());
     }
 
     // can not use constructor injection or there will be a circular dependency
@@ -165,13 +166,14 @@ public class NodeService extends AbstractComponent implements Closeable {
                 httpServer == null ? null : httpServer.stats(),
                 circuitBreakerService.stats(),
                 scriptService.stats(),
-                discovery.stats()
+                discovery.stats(),
+                ingestService.getPipelineExecutionService().stats()
         );
     }
 
     public NodeStats stats(CommonStatsFlags indices, boolean os, boolean process, boolean jvm, boolean threadPool,
                            boolean fs, boolean transport, boolean http, boolean circuitBreaker,
-                           boolean script, boolean discoveryStats) {
+                           boolean script, boolean discoveryStats, boolean ingest) {
         // for indices stats we want to include previous allocated shards stats as well (it will
         // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats)
         return new NodeStats(discovery.localNode(), System.currentTimeMillis(),
@@ -185,7 +187,8 @@ public class NodeService extends AbstractComponent implements Closeable {
                 http ? (httpServer == null ? null : httpServer.stats()) : null,
                 circuitBreaker ? circuitBreakerService.stats() : null,
                 script ? scriptService.stats() : null,
-                discoveryStats ? discovery.stats() : null
+                discoveryStats ? discovery.stats() : null,
+                ingest ? ingestService.getPipelineExecutionService().stats() : null
         );
     }
 
diff --git a/core/src/main/java/org/elasticsearch/percolator/MultiDocumentPercolatorIndex.java b/core/src/main/java/org/elasticsearch/percolator/MultiDocumentPercolatorIndex.java
deleted file mode 100644
index 9d091a4c0bd..00000000000
--- a/core/src/main/java/org/elasticsearch/percolator/MultiDocumentPercolatorIndex.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.elasticsearch.percolator;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.MultiReader;
-import org.apache.lucene.index.SlowCompositeReaderWrapper;
-import org.apache.lucene.index.memory.MemoryIndex;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.util.CloseableThreadLocal;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ParseContext;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.internal.UidFieldMapper;
-
-import java.io.IOException;
-import java.util.List;
-
-
-/**
- * Implementation of {@link PercolatorIndex} that can hold multiple Lucene documents by
- * opening multiple {@link MemoryIndex} based IndexReaders and wrapping them via a single top level reader.
- */
-class MultiDocumentPercolatorIndex implements PercolatorIndex {
-
-    private final CloseableThreadLocal<MemoryIndex> cache;
-
-    MultiDocumentPercolatorIndex(CloseableThreadLocal<MemoryIndex> cache) {
-        this.cache = cache;
-    }
-
-    @Override
-    public void prepare(PercolateContext context, ParsedDocument parsedDocument) {
-        IndexReader[] memoryIndices = new IndexReader[parsedDocument.docs().size()];
-        List<ParseContext.Document> docs = parsedDocument.docs();
-        int rootDocIndex = docs.size() - 1;
-        assert rootDocIndex > 0;
-        MemoryIndex rootDocMemoryIndex = null;
-        for (int i = 0; i < docs.size(); i++) {
-            ParseContext.Document d = docs.get(i);
-            MemoryIndex memoryIndex;
-            if (rootDocIndex == i) {
-                // the last doc is always the rootDoc, since that is usually the biggest document it make sense
-                // to reuse the MemoryIndex it uses
-                memoryIndex = rootDocMemoryIndex = cache.get();
-            } else {
-                memoryIndex = new MemoryIndex(true);
-            }
-            memoryIndices[i] = indexDoc(d, memoryIndex, context, parsedDocument).createSearcher().getIndexReader();
-        }
-        try {
-            MultiReader mReader = new MultiReader(memoryIndices, true);
-            LeafReader slowReader = SlowCompositeReaderWrapper.wrap(mReader);
-            final IndexSearcher slowSearcher = new IndexSearcher(slowReader) {
-
-                @Override
-                public Weight createNormalizedWeight(Query query, boolean needsScores) throws IOException {
-                    BooleanQuery.Builder bq = new BooleanQuery.Builder();
-                    bq.add(query, BooleanClause.Occur.MUST);
-                    bq.add(Queries.newNestedFilter(), BooleanClause.Occur.MUST_NOT);
-                    return super.createNormalizedWeight(bq.build(), needsScores);
-                }
-
-            };
-            slowSearcher.setQueryCache(null);
-            DocSearcher docSearcher = new DocSearcher(slowSearcher, rootDocMemoryIndex);
-            context.initialize(docSearcher, parsedDocument);
-        } catch (IOException e) {
-            throw new ElasticsearchException("Failed to create index for percolator with nested document ", e);
-        }
-    }
-
-    MemoryIndex indexDoc(ParseContext.Document d, MemoryIndex memoryIndex, PercolateContext context, ParsedDocument parsedDocument) {
-        for (IndexableField field : d.getFields()) {
-            Analyzer analyzer = context.analysisService().defaultIndexAnalyzer();
-            DocumentMapper documentMapper = context.mapperService().documentMapper(parsedDocument.type());
-            if (documentMapper != null && documentMapper.mappers().getMapper(field.name()) != null) {
-                analyzer =  documentMapper.mappers().indexAnalyzer();
-            }
-            if (field.fieldType().indexOptions() == IndexOptions.NONE && field.name().equals(UidFieldMapper.NAME)) {
-                continue;
-            }
-            try {
-                // TODO: instead of passing null here, we can have a CTL<Map<String,TokenStream>> and pass previous,
-                // like the indexer does
-                try (TokenStream tokenStream = field.tokenStream(analyzer, null)) {
-                    if (tokenStream != null) {
-                        memoryIndex.addField(field.name(), tokenStream, field.boost());
-                    }
-                 }
-            } catch (IOException e) {
-                throw new ElasticsearchException("Failed to create token stream", e);
-            }
-        }
-        return memoryIndex;
-    }
-
-    private class DocSearcher extends Engine.Searcher {
-
-        private final MemoryIndex rootDocMemoryIndex;
-
-        private DocSearcher(IndexSearcher searcher, MemoryIndex rootDocMemoryIndex) {
-            super("percolate", searcher);
-            this.rootDocMemoryIndex = rootDocMemoryIndex;
-        }
-
-        @Override
-        public void close() {
-            try {
-                this.reader().close();
-                rootDocMemoryIndex.reset();
-            } catch (IOException e) {
-                throw new ElasticsearchException("failed to close IndexReader in percolator with nested doc", e);
-            }
-        }
-
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/percolator/PercolateContext.java b/core/src/main/java/org/elasticsearch/percolator/PercolateContext.java
deleted file mode 100644
index f73c8f31a07..00000000000
--- a/core/src/main/java/org/elasticsearch/percolator/PercolateContext.java
+++ /dev/null
@@ -1,691 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.percolator;
-
-
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.FieldDoc;
-import org.apache.lucene.util.Counter;
-import org.elasticsearch.action.percolate.PercolateShardRequest;
-import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.lease.Releasables;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.analysis.AnalysisService;
-import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
-import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.fielddata.IndexFieldDataService;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.object.ObjectMapper;
-import org.elasticsearch.index.query.ParsedQuery;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.similarity.SimilarityService;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchHitField;
-import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.aggregations.SearchContextAggregations;
-import org.elasticsearch.search.dfs.DfsSearchResult;
-import org.elasticsearch.search.fetch.FetchPhase;
-import org.elasticsearch.search.fetch.FetchSearchResult;
-import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.fetch.FetchSubPhaseContext;
-import org.elasticsearch.search.fetch.innerhits.InnerHitsContext;
-import org.elasticsearch.search.fetch.script.ScriptFieldsContext;
-import org.elasticsearch.search.fetch.source.FetchSourceContext;
-import org.elasticsearch.search.highlight.SearchContextHighlight;
-import org.elasticsearch.search.internal.ContextIndexSearcher;
-import org.elasticsearch.search.internal.InternalSearchHit;
-import org.elasticsearch.search.internal.InternalSearchHitField;
-import org.elasticsearch.search.internal.ScrollContext;
-import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.internal.ShardSearchRequest;
-import org.elasticsearch.search.lookup.LeafSearchLookup;
-import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.profile.Profilers;
-import org.elasticsearch.search.query.QuerySearchResult;
-import org.elasticsearch.search.rescore.RescoreSearchContext;
-import org.elasticsearch.search.suggest.SuggestionSearchContext;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- */
-public class PercolateContext extends SearchContext {
-
-    private int size = 10;
-    private boolean trackScores;
-
-    private final SearchShardTarget searchShardTarget;
-    private final IndexService indexService;
-    private final IndexFieldDataService fieldDataService;
-    private final IndexShard indexShard;
-    private final PageCacheRecycler pageCacheRecycler;
-    private final BigArrays bigArrays;
-    private final ScriptService scriptService;
-    private final MapperService mapperService;
-    private final int numberOfShards;
-    private final Query aliasFilter;
-    private final long originNanoTime = System.nanoTime();
-    private final long startTime;
-    private final boolean onlyCount;
-    private Engine.Searcher docSearcher;
-    private Engine.Searcher engineSearcher;
-    private ContextIndexSearcher searcher;
-
-    private SearchContextHighlight highlight;
-    private ParsedQuery parsedQuery;
-    private Query query;
-    private Query percolateQuery;
-    private FetchSubPhase.HitContext hitContext;
-    private SearchContextAggregations aggregations;
-    private QuerySearchResult querySearchResult;
-    private Sort sort;
-    private final Map<String, FetchSubPhaseContext> subPhaseContexts = new HashMap<>();
-    private final QueryShardContext queryShardContext;
-    private final Map<Class<?>, Collector> queryCollectors = new HashMap<>();
-    private SearchLookup searchLookup;
-    private final FetchPhase fetchPhase;
-
-    public PercolateContext(PercolateShardRequest request, SearchShardTarget searchShardTarget, IndexShard indexShard,
-            IndexService indexService, PageCacheRecycler pageCacheRecycler, BigArrays bigArrays, ScriptService scriptService,
-            Query aliasFilter, ParseFieldMatcher parseFieldMatcher, FetchPhase fetchPhase) {
-        super(parseFieldMatcher);
-        this.indexShard = indexShard;
-        this.indexService = indexService;
-        this.fetchPhase = fetchPhase;
-        this.fieldDataService = indexService.fieldData();
-        this.mapperService = indexService.mapperService();
-        this.searchShardTarget = searchShardTarget;
-        this.pageCacheRecycler = pageCacheRecycler;
-        this.bigArrays = bigArrays.withCircuitBreaking();
-        this.querySearchResult = new QuerySearchResult(0, searchShardTarget);
-        this.engineSearcher = indexShard.acquireSearcher("percolate");
-        this.searcher = new ContextIndexSearcher(engineSearcher, indexService.cache().query(), indexShard.getQueryCachingPolicy());
-        this.scriptService = scriptService;
-        this.numberOfShards = request.getNumberOfShards();
-        this.aliasFilter = aliasFilter;
-        this.startTime = request.getStartTime();
-        this.onlyCount = request.onlyCount();
-        queryShardContext = indexService.newQueryShardContext();
-        queryShardContext.setTypes(request.documentType());
-    }
-
-    // for testing:
-    PercolateContext(PercolateShardRequest request, SearchShardTarget searchShardTarget, MapperService mapperService, QueryShardContext queryShardContext) {
-        super(null);
-        this.searchShardTarget = searchShardTarget;
-        this.mapperService = mapperService;
-        this.indexService = null;
-        this.indexShard = null;
-        this.fieldDataService = null;
-        this.pageCacheRecycler = null;
-        this.bigArrays = null;
-        this.scriptService = null;
-        this.aliasFilter = null;
-        this.startTime = 0;
-        this.numberOfShards = 0;
-        this.onlyCount = true;
-        this.queryShardContext = queryShardContext;
-        this.fetchPhase = null;
-    }
-
-    public IndexSearcher docSearcher() {
-        return docSearcher.searcher();
-    }
-
-    public void initialize(Engine.Searcher docSearcher, ParsedDocument parsedDocument) {
-        this.docSearcher = docSearcher;
-        IndexReader indexReader = docSearcher.reader();
-        LeafReaderContext atomicReaderContext = indexReader.leaves().get(0);
-        this.searchLookup = new SearchLookup(mapperService(), fieldData(),  queryShardContext.getTypes());
-        LeafSearchLookup leafLookup = searchLookup.getLeafSearchLookup(atomicReaderContext);
-        leafLookup.setDocument(0);
-        leafLookup.source().setSource(parsedDocument.source());
-
-        Map<String, SearchHitField> fields = new HashMap<>();
-        for (IndexableField field : parsedDocument.rootDoc().getFields()) {
-            fields.put(field.name(), new InternalSearchHitField(field.name(), Collections.emptyList()));
-        }
-        hitContext().reset(
-                new InternalSearchHit(0, "unknown", new Text(parsedDocument.type()), fields),
-                atomicReaderContext, 0, docSearcher.searcher()
-        );
-    }
-
-    @Override
-    public IndexShard indexShard() {
-        return indexShard;
-    }
-
-    public IndexService indexService() {
-        return indexService;
-    }
-
-    public Query percolateQuery() {
-        return percolateQuery;
-    }
-
-    public void percolateQuery(Query percolateQuery) {
-        this.percolateQuery = percolateQuery;
-    }
-
-    public FetchSubPhase.HitContext hitContext() {
-        if (hitContext == null) {
-            hitContext = new FetchSubPhase.HitContext();
-        }
-        return hitContext;
-    }
-
-    public boolean isOnlyCount() {
-        return onlyCount;
-    }
-
-    public Query percolatorTypeFilter(){
-        return indexService().mapperService().documentMapper(PercolatorService.TYPE_NAME).typeFilter();
-    }
-
-    @Override
-    public SearchContextHighlight highlight() {
-        return highlight;
-    }
-
-    @Override
-    public void highlight(SearchContextHighlight highlight) {
-        if (highlight != null) {
-            // Enforce highlighting by source, because MemoryIndex doesn't support stored fields.
-            highlight.globalForceSource(true);
-        }
-        this.highlight = highlight;
-    }
-
-    @Override
-    public SearchShardTarget shardTarget() {
-        return searchShardTarget;
-    }
-
-    @Override
-    public SearchLookup lookup() {
-        // we cache this since it's really just a single document lookup - check the init method for details
-        assert searchLookup != null : "context is not initialized";
-        assert Arrays.equals(searchLookup.doc().getTypes(), getQueryShardContext().getTypes()) : "types mismatch - can't return lookup";
-        return this.searchLookup;
-    }
-
-    @Override
-    protected void doClose() {
-        Releasables.close(engineSearcher, docSearcher);
-    }
-
-    @Override
-    public MapperService mapperService() {
-        return mapperService;
-    }
-
-    @Override
-    public SearchContext parsedQuery(ParsedQuery query) {
-        this.parsedQuery = query;
-        this.query = query.query();
-        return this;
-    }
-
-    @Override
-    public ParsedQuery parsedQuery() {
-        return parsedQuery;
-    }
-
-    @Override
-    public Query query() {
-        return query;
-    }
-
-    @Override
-    public IndexFieldDataService fieldData() {
-        return fieldDataService;
-    }
-
-    @Override
-    public SearchContextAggregations aggregations() {
-        return aggregations;
-    }
-
-    @Override
-    public SearchContext aggregations(SearchContextAggregations aggregations) {
-        this.aggregations = aggregations;
-        return this;
-    }
-
-    @Override
-    public <SubPhaseContext extends FetchSubPhaseContext> SubPhaseContext getFetchSubPhaseContext(FetchSubPhase.ContextFactory<SubPhaseContext> contextFactory) {
-        String subPhaseName = contextFactory.getName();
-        if (subPhaseContexts.get(subPhaseName) == null) {
-            subPhaseContexts.put(subPhaseName, contextFactory.newContextInstance());
-        }
-        return (SubPhaseContext) subPhaseContexts.get(subPhaseName);
-    }
-
-    // Unused:
-    @Override
-    public void preProcess() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public Query searchFilter(String[] types) {
-        return aliasFilter();
-    }
-
-    @Override
-    public long id() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public String source() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public ShardSearchRequest request() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public SearchType searchType() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public SearchContext searchType(SearchType searchType) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public int numberOfShards() {
-        return numberOfShards;
-    }
-
-    @Override
-    public float queryBoost() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public SearchContext queryBoost(float queryBoost) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public long getOriginNanoTime() {
-        return originNanoTime;
-    }
-
-    @Override
-    protected long nowInMillisImpl() {
-        return startTime;
-    }
-
-    @Override
-    public ScrollContext scrollContext() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public SearchContext scrollContext(ScrollContext scroll) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public SuggestionSearchContext suggest() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void suggest(SuggestionSearchContext suggest) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public List<RescoreSearchContext> rescore() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void addRescore(RescoreSearchContext rescore) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public boolean hasScriptFields() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public ScriptFieldsContext scriptFields() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public boolean sourceRequested() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public boolean hasFetchSourceContext() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public FetchSourceContext fetchSourceContext() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public SearchContext fetchSourceContext(FetchSourceContext fetchSourceContext) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public ContextIndexSearcher searcher() {
-        return searcher;
-    }
-
-    @Override
-    public AnalysisService analysisService() {
-        return indexService.analysisService();
-    }
-
-    @Override
-    public SimilarityService similarityService() {
-        return indexService.similarityService();
-    }
-
-    @Override
-    public ScriptService scriptService() {
-        return scriptService;
-    }
-
-    @Override
-    public PageCacheRecycler pageCacheRecycler() {
-        return pageCacheRecycler;
-    }
-
-    @Override
-    public BigArrays bigArrays() {
-        return bigArrays;
-    }
-
-    @Override
-    public BitsetFilterCache bitsetFilterCache() {
-        return indexService.cache().bitsetFilterCache();
-    }
-
-    @Override
-    public long timeoutInMillis() {
-        return -1;
-    }
-
-    @Override
-    public void timeoutInMillis(long timeoutInMillis) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public int terminateAfter() {
-        return DEFAULT_TERMINATE_AFTER;
-    }
-
-    @Override
-    public void terminateAfter(int terminateAfter) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public SearchContext minimumScore(float minimumScore) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public Float minimumScore() {
-        return null;
-    }
-
-    @Override
-    public SearchContext sort(Sort sort) {
-        this.sort = sort;
-        return this;
-    }
-
-    @Override
-    public Sort sort() {
-        return sort;
-    }
-
-    @Override
-    public SearchContext trackScores(boolean trackScores) {
-        this.trackScores = trackScores;
-        return this;
-    }
-
-    @Override
-    public boolean trackScores() {
-        return trackScores;
-    }
-
-    @Override
-    public SearchContext searchAfter(FieldDoc searchAfter) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public FieldDoc searchAfter() {
-        return null;
-    }
-
-    @Override
-    public SearchContext parsedPostFilter(ParsedQuery postFilter) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public ParsedQuery parsedPostFilter() {
-        return null;
-    }
-
-    @Override
-    public Query aliasFilter() {
-        return aliasFilter;
-    }
-
-    @Override
-    public int from() {
-        return 0;
-    }
-
-    @Override
-    public SearchContext from(int from) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public int size() {
-        return size;
-    }
-
-    @Override
-    public SearchContext size(int size) {
-        this.size = size;
-        return this;
-    }
-
-    @Override
-    public boolean hasFieldNames() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public List<String> fieldNames() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void emptyFieldNames() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public boolean explain() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void explain(boolean explain) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public List<String> groupStats() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void groupStats(List<String> groupStats) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public boolean version() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void version(boolean version) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public int[] docIdsToLoad() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public int docIdsToLoadFrom() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public int docIdsToLoadSize() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public SearchContext docIdsToLoad(int[] docIdsToLoad, int docsIdsToLoadFrom, int docsIdsToLoadSize) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void accessed(long accessTime) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public long lastAccessTime() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public long keepAlive() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void keepAlive(long keepAlive) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public DfsSearchResult dfsResult() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public QuerySearchResult queryResult() {
-        return querySearchResult;
-    }
-
-    @Override
-    public FetchSearchResult fetchResult() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public FetchPhase fetchPhase() {
-        return fetchPhase;
-    }
-
-    @Override
-    public MappedFieldType smartNameFieldType(String name) {
-        return mapperService().fullName(name);
-    }
-
-    @Override
-    public ObjectMapper getObjectMapper(String name) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public Counter timeEstimateCounter() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public InnerHitsContext innerHits() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public Map<Class<?>, Collector> queryCollectors() {
-        return queryCollectors;
-    }
-
-    @Override
-    public QueryShardContext getQueryShardContext() {
-        return queryShardContext;
-    }
-
-    @Override
-    public Profilers getProfilers() {
-        throw new UnsupportedOperationException();
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/percolator/PercolateDocumentParser.java b/core/src/main/java/org/elasticsearch/percolator/PercolateDocumentParser.java
deleted file mode 100644
index 50db3cecaa6..00000000000
--- a/core/src/main/java/org/elasticsearch/percolator/PercolateDocumentParser.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.percolator;
-
-import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.Query;
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.action.percolate.PercolateShardRequest;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.mapper.DocumentMapperForType;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.aggregations.AggregationPhase;
-import org.elasticsearch.search.highlight.HighlightPhase;
-import org.elasticsearch.search.sort.SortParseElement;
-
-import java.util.Map;
-
-import static org.elasticsearch.index.mapper.SourceToParse.source;
-
-public class PercolateDocumentParser {
-
-    private final HighlightPhase highlightPhase;
-    private final SortParseElement sortParseElement;
-    private final AggregationPhase aggregationPhase;
-
-    @Inject
-    public PercolateDocumentParser(HighlightPhase highlightPhase, SortParseElement sortParseElement,
-                                   AggregationPhase aggregationPhase) {
-        this.highlightPhase = highlightPhase;
-        this.sortParseElement = sortParseElement;
-        this.aggregationPhase = aggregationPhase;
-    }
-
-    public ParsedDocument parse(final PercolateShardRequest request, final PercolateContext context, final MapperService mapperService) {
-        BytesReference source = request.source();
-        if (source == null || source.length() == 0) {
-            if (request.docSource() != null && request.docSource().length() != 0) {
-                return parseFetchedDoc(context, request.docSource(), mapperService, request.shardId().getIndexName(), request.documentType());
-            } else {
-                return null;
-            }
-        }
-
-        // TODO: combine all feature parse elements into one map
-        Map<String, ? extends SearchParseElement> hlElements = highlightPhase.parseElements();
-        Map<String, ? extends SearchParseElement> aggregationElements = aggregationPhase.parseElements();
-        final QueryShardContext queryShardContext = context.getQueryShardContext();
-        ParsedDocument doc = null;
-        // Some queries (function_score query when for decay functions) rely on a SearchContext being set:
-        // We switch types because this context needs to be in the context of the percolate queries in the shard and
-        // not the in memory percolate doc
-        final String[] previousTypes = queryShardContext.getTypes();
-        queryShardContext.setTypes(PercolatorService.TYPE_NAME);
-        try (XContentParser parser = XContentFactory.xContent(source).createParser(source);) {
-            String currentFieldName = null;
-            XContentParser.Token token;
-            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    currentFieldName = parser.currentName();
-                    // we need to check the "doc" here, so the next token will be START_OBJECT which is
-                    // the actual document starting
-                    if ("doc".equals(currentFieldName)) {
-                        if (doc != null) {
-                            throw new ElasticsearchParseException("Either specify doc or get, not both");
-                        }
-
-                        DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(request.documentType());
-                        String index = context.shardTarget().index();
-                        doc = docMapper.getDocumentMapper().parse(source(parser).index(index).type(request.documentType()).id("_id_for_percolate_api"));
-                        if (docMapper.getMapping() != null) {
-                            doc.addDynamicMappingsUpdate(docMapper.getMapping());
-                        }
-                        // the document parsing exists the "doc" object, so we need to set the new current field.
-                        currentFieldName = parser.currentName();
-                    }
-                } else if (token == XContentParser.Token.START_OBJECT) {
-                    SearchParseElement element = hlElements.get(currentFieldName);
-                    if (element == null) {
-                        element = aggregationElements.get(currentFieldName);
-                    }
-
-                    if ("query".equals(currentFieldName)) {
-                        if (context.percolateQuery() != null) {
-                            throw new ElasticsearchParseException("Either specify query or filter, not both");
-                        }
-                        context.percolateQuery(queryShardContext.parse(parser).query());
-                    } else if ("filter".equals(currentFieldName)) {
-                        if (context.percolateQuery() != null) {
-                            throw new ElasticsearchParseException("Either specify query or filter, not both");
-                        }
-                        Query filter = queryShardContext.parseInnerFilter(parser).query();
-                        context.percolateQuery(new ConstantScoreQuery(filter));
-                    } else if ("sort".equals(currentFieldName)) {
-                        parseSort(parser, context);
-                    } else if (element != null) {
-                        element.parse(parser, context);
-                    }
-                } else if (token == XContentParser.Token.START_ARRAY) {
-                    if ("sort".equals(currentFieldName)) {
-                        parseSort(parser, context);
-                    }
-                } else if (token == null) {
-                    break;
-                } else if (token.isValue()) {
-                    if ("size".equals(currentFieldName)) {
-                        context.size(parser.intValue());
-                        if (context.size() < 0) {
-                            throw new ElasticsearchParseException("size is set to [{}] and is expected to be higher or equal to 0", context.size());
-                        }
-                    } else if ("sort".equals(currentFieldName)) {
-                        parseSort(parser, context);
-                    } else if ("track_scores".equals(currentFieldName) || "trackScores".equals(currentFieldName)) {
-                        context.trackScores(parser.booleanValue());
-                    }
-                }
-            }
-
-            // We need to get the actual source from the request body for highlighting, so parse the request body again
-            // and only get the doc source.
-            if (context.highlight() != null) {
-                parser.close();
-                currentFieldName = null;
-                try (XContentParser parserForHighlighter = XContentFactory.xContent(source).createParser(source)) {
-                    token = parserForHighlighter.nextToken();
-                    assert token == XContentParser.Token.START_OBJECT;
-                    while ((token = parserForHighlighter.nextToken()) != XContentParser.Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            currentFieldName = parserForHighlighter.currentName();
-                        } else if (token == XContentParser.Token.START_OBJECT) {
-                            if ("doc".equals(currentFieldName)) {
-                                BytesStreamOutput bStream = new BytesStreamOutput();
-                                XContentBuilder builder = XContentFactory.contentBuilder(XContentType.SMILE, bStream);
-                                builder.copyCurrentStructure(parserForHighlighter);
-                                builder.close();
-                                doc.setSource(bStream.bytes());
-                                break;
-                            } else {
-                                parserForHighlighter.skipChildren();
-                            }
-                        } else if (token == null) {
-                            break;
-                        }
-                    }
-                }
-            }
-
-        } catch (Throwable e) {
-            throw new ElasticsearchParseException("failed to parse request", e);
-        } finally {
-            queryShardContext.setTypes(previousTypes);
-        }
-
-        if (request.docSource() != null && request.docSource().length() != 0) {
-            if (doc != null) {
-                throw new IllegalArgumentException("Can't specify the document to percolate in the source of the request and as document id");
-            }
-
-            doc = parseFetchedDoc(context, request.docSource(), mapperService, request.shardId().getIndexName(), request.documentType());
-        }
-
-        if (doc == null) {
-            throw new IllegalArgumentException("Nothing to percolate");
-        }
-
-        return doc;
-    }
-
-    private void parseSort(XContentParser parser, PercolateContext context) throws Exception {
-        context.trackScores(true);
-        sortParseElement.parse(parser, context);
-        // null, means default sorting by relevancy
-        if (context.sort() != null) {
-            throw new ElasticsearchParseException("Only _score desc is supported");
-        }
-    }
-
-    private ParsedDocument parseFetchedDoc(PercolateContext context, BytesReference fetchedDoc, MapperService mapperService, String index, String type) {
-        DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(type);
-        ParsedDocument doc = docMapper.getDocumentMapper().parse(source(fetchedDoc).index(index).type(type).id("_id_for_percolate_api"));
-        if (doc == null) {
-            throw new ElasticsearchParseException("No doc to percolate in the request");
-        }
-        if (context.highlight() != null) {
-            doc.setSource(fetchedDoc);
-        }
-        return doc;
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/percolator/PercolateException.java b/core/src/main/java/org/elasticsearch/percolator/PercolateException.java
deleted file mode 100644
index 81a708a75ec..00000000000
--- a/core/src/main/java/org/elasticsearch/percolator/PercolateException.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ElasticsearchWrapperException;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.index.shard.ShardId;
-
-import java.io.IOException;
-import java.util.Objects;
-
-/**
- * Exception during percolating document(s) at runtime.
- */
-public class PercolateException extends ElasticsearchException implements ElasticsearchWrapperException {
-
-    private final ShardId shardId;
-
-    public PercolateException(ShardId shardId, String msg, Throwable cause) {
-        super(msg, cause);
-        Objects.requireNonNull(shardId, "shardId must not be null");
-        this.shardId = shardId;
-    }
-
-    public ShardId getShardId() {
-        return shardId;
-    }
-
-    public PercolateException(StreamInput in) throws IOException{
-        super(in);
-        shardId = ShardId.readShardId(in);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        shardId.writeTo(out);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java b/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java
deleted file mode 100644
index 1160aec969b..00000000000
--- a/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java
+++ /dev/null
@@ -1,376 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.percolator;
-
-
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.ReaderUtil;
-import org.apache.lucene.index.memory.ExtendedMemoryIndex;
-import org.apache.lucene.index.memory.MemoryIndex;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.MultiCollector;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.TopScoreDocCollector;
-import org.apache.lucene.search.TotalHitCountCollector;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.CloseableThreadLocal;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.percolate.PercolateResponse;
-import org.elasticsearch.action.percolate.PercolateShardRequest;
-import org.elasticsearch.action.percolate.PercolateShardResponse;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
-import org.elasticsearch.cluster.ClusterService;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.lease.Releasable;
-import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.fieldvisitor.SingleFieldsVisitor;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.internal.UidFieldMapper;
-import org.elasticsearch.index.percolator.PercolatorFieldMapper;
-import org.elasticsearch.index.percolator.PercolatorQueriesRegistry;
-import org.elasticsearch.index.query.ParsedQuery;
-import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.aggregations.AggregationPhase;
-import org.elasticsearch.search.aggregations.Aggregator;
-import org.elasticsearch.search.aggregations.BucketCollector;
-import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregator;
-import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.fetch.FetchPhase;
-import org.elasticsearch.search.highlight.HighlightField;
-import org.elasticsearch.search.highlight.HighlightPhase;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
-
-import static org.apache.lucene.search.BooleanClause.Occur.FILTER;
-import static org.apache.lucene.search.BooleanClause.Occur.MUST;
-
-public class PercolatorService extends AbstractComponent implements Releasable {
-
-    public final static float NO_SCORE = Float.NEGATIVE_INFINITY;
-    public final static String TYPE_NAME = ".percolator";
-
-    private final BigArrays bigArrays;
-    private final ScriptService scriptService;
-    private final IndicesService indicesService;
-    private final ClusterService clusterService;
-    private final HighlightPhase highlightPhase;
-    private final AggregationPhase aggregationPhase;
-    private final PageCacheRecycler pageCacheRecycler;
-    private final CloseableThreadLocal<MemoryIndex> cache;
-    private final IndexNameExpressionResolver indexNameExpressionResolver;
-    private final PercolateDocumentParser percolateDocumentParser;
-
-    private final PercolatorIndex single;
-    private final PercolatorIndex multi;
-    private final ParseFieldMatcher parseFieldMatcher;
-    private final FetchPhase fetchPhase;
-
-    @Inject
-    public PercolatorService(Settings settings, IndexNameExpressionResolver indexNameExpressionResolver, IndicesService indicesService,
-                             PageCacheRecycler pageCacheRecycler, BigArrays bigArrays,
-                             HighlightPhase highlightPhase, ClusterService clusterService,
-                             AggregationPhase aggregationPhase, ScriptService scriptService,
-                             PercolateDocumentParser percolateDocumentParser, FetchPhase fetchPhase) {
-        super(settings);
-        this.indexNameExpressionResolver = indexNameExpressionResolver;
-        this.percolateDocumentParser = percolateDocumentParser;
-        this.fetchPhase = fetchPhase;
-        this.parseFieldMatcher = new ParseFieldMatcher(settings);
-        this.indicesService = indicesService;
-        this.pageCacheRecycler = pageCacheRecycler;
-        this.bigArrays = bigArrays;
-        this.clusterService = clusterService;
-        this.scriptService = scriptService;
-        this.aggregationPhase = aggregationPhase;
-        this.highlightPhase = highlightPhase;
-
-        final long maxReuseBytes = settings.getAsBytesSize("indices.memory.memory_index.size_per_thread", new ByteSizeValue(1, ByteSizeUnit.MB)).bytes();
-        cache = new CloseableThreadLocal<MemoryIndex>() {
-            @Override
-            protected MemoryIndex initialValue() {
-                // TODO: should we expose payloads as an option? should offsets be turned on always?
-                return new ExtendedMemoryIndex(true, false, maxReuseBytes);
-            }
-        };
-        single = new SingleDocumentPercolatorIndex(cache);
-        multi = new MultiDocumentPercolatorIndex(cache);
-    }
-
-    public ReduceResult reduce(boolean onlyCount, List<PercolateShardResponse> shardResponses) throws IOException {
-        if (onlyCount) {
-            long finalCount = 0;
-            for (PercolateShardResponse shardResponse : shardResponses) {
-                finalCount += shardResponse.topDocs().totalHits;
-            }
-
-            InternalAggregations reducedAggregations = reduceAggregations(shardResponses);
-            return new PercolatorService.ReduceResult(finalCount, reducedAggregations);
-        } else {
-            int requestedSize = shardResponses.get(0).requestedSize();
-            TopDocs[] shardResults = new TopDocs[shardResponses.size()];
-            long foundMatches = 0;
-            for (int i = 0; i < shardResults.length; i++) {
-                TopDocs shardResult = shardResponses.get(i).topDocs();
-                foundMatches += shardResult.totalHits;
-                shardResults[i] = shardResult;
-            }
-            TopDocs merged = TopDocs.merge(requestedSize, shardResults);
-            PercolateResponse.Match[] matches = new PercolateResponse.Match[merged.scoreDocs.length];
-            for (int i = 0; i < merged.scoreDocs.length; i++) {
-                ScoreDoc doc = merged.scoreDocs[i];
-                PercolateShardResponse shardResponse = shardResponses.get(doc.shardIndex);
-                String id = shardResponse.ids().get(doc.doc);
-                Map<String, HighlightField> hl = shardResponse.hls().get(doc.doc);
-                matches[i] = new PercolateResponse.Match(new Text(shardResponse.getIndex()), new Text(id), doc.score, hl);
-            }
-            InternalAggregations reducedAggregations = reduceAggregations(shardResponses);
-            return new PercolatorService.ReduceResult(foundMatches, matches, reducedAggregations);
-        }
-    }
-
-    public PercolateShardResponse percolate(PercolateShardRequest request) throws IOException {
-        final IndexService percolateIndexService = indicesService.indexServiceSafe(request.shardId().getIndex());
-        final IndexShard indexShard = percolateIndexService.getShard(request.shardId().id());
-        indexShard.readAllowed(); // check if we can read the shard...
-        PercolatorQueriesRegistry percolateQueryRegistry = indexShard.percolateRegistry();
-        percolateQueryRegistry.prePercolate();
-        long startTime = System.nanoTime();
-
-        // TODO: The filteringAliases should be looked up at the coordinating node and serialized with all shard request,
-        // just like is done in other apis.
-        String[] filteringAliases = indexNameExpressionResolver.filteringAliases(
-                clusterService.state(),
-                indexShard.shardId().getIndex().getName(),
-                request.indices()
-        );
-        Query aliasFilter = percolateIndexService.aliasFilter(percolateIndexService.newQueryShardContext(), filteringAliases);
-
-        SearchShardTarget searchShardTarget = new SearchShardTarget(clusterService.localNode().id(), request.shardId().getIndex(),
-                request.shardId().id());
-        final PercolateContext context = new PercolateContext(request, searchShardTarget, indexShard, percolateIndexService,
-                pageCacheRecycler, bigArrays, scriptService, aliasFilter, parseFieldMatcher, fetchPhase);
-        SearchContext.setCurrent(context);
-        try {
-            ParsedDocument parsedDocument = percolateDocumentParser.parse(request, context, percolateIndexService.mapperService());
-            if (context.searcher().getIndexReader().maxDoc() == 0) {
-                return new PercolateShardResponse(Lucene.EMPTY_TOP_DOCS, Collections.emptyMap(), Collections.emptyMap(), context);
-            }
-            if (context.size() < 0) {
-                context.size(0);
-            }
-
-            // parse the source either into one MemoryIndex, if it is a single document or index multiple docs if nested
-            PercolatorIndex percolatorIndex;
-            DocumentMapper documentMapper = indexShard.mapperService().documentMapper(request.documentType());
-            boolean isNested = documentMapper != null && documentMapper.hasNestedObjects();
-            if (parsedDocument.docs().size() > 1) {
-                assert isNested;
-                percolatorIndex = multi;
-            } else {
-                percolatorIndex = single;
-            }
-            percolatorIndex.prepare(context, parsedDocument);
-
-            BucketCollector aggregatorCollector = null;
-            if (context.aggregations() != null) {
-                AggregationContext aggregationContext = new AggregationContext(context);
-                context.aggregations().aggregationContext(aggregationContext);
-                Aggregator[] aggregators = context.aggregations().factories().createTopLevelAggregators();
-                List<Aggregator> aggregatorCollectors = new ArrayList<>(aggregators.length);
-                for (int i = 0; i < aggregators.length; i++) {
-                    if (!(aggregators[i] instanceof GlobalAggregator)) {
-                        Aggregator aggregator = aggregators[i];
-                        aggregatorCollectors.add(aggregator);
-                }
-            }
-                context.aggregations().aggregators(aggregators);
-                aggregatorCollector = BucketCollector.wrap(aggregatorCollectors);
-                aggregatorCollector.preCollection();
-            }
-            PercolatorQueriesRegistry queriesRegistry = indexShard.percolateRegistry();
-            return doPercolate(context, queriesRegistry, aggregationPhase, aggregatorCollector, highlightPhase);
-        } finally {
-            SearchContext.removeCurrent();
-            context.close();
-            percolateQueryRegistry.postPercolate(System.nanoTime() - startTime);
-        }
-    }
-
-    // moved the core percolation logic to a pck protected method to make testing easier:
-    static PercolateShardResponse doPercolate(PercolateContext context, PercolatorQueriesRegistry queriesRegistry, AggregationPhase aggregationPhase, @Nullable BucketCollector aggregatorCollector, HighlightPhase highlightPhase) throws IOException {
-        PercolatorQuery.Builder builder = new PercolatorQuery.Builder(context.docSearcher(), queriesRegistry.getPercolateQueries(), context.percolatorTypeFilter());
-        if (queriesRegistry.indexSettings().getSettings().getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null).onOrAfter(Version.V_5_0_0)) {
-            builder.extractQueryTermsQuery(PercolatorFieldMapper.EXTRACTED_TERMS_FULL_FIELD_NAME, PercolatorFieldMapper.UNKNOWN_QUERY_FULL_FIELD_NAME);
-        }
-        if (context.percolateQuery() != null || context.aliasFilter() != null) {
-            BooleanQuery.Builder bq = new BooleanQuery.Builder();
-                        if (context.percolateQuery() != null) {
-                bq.add(context.percolateQuery(), MUST);
-                        }
-            if (context.aliasFilter() != null) {
-                bq.add(context.aliasFilter(), FILTER);
-                        }
-            builder.setPercolateQuery(bq.build());
-                    }
-        PercolatorQuery percolatorQuery = builder.build();
-
-        if (context.isOnlyCount() || context.size() == 0) {
-            TotalHitCountCollector collector = new TotalHitCountCollector();
-            context.searcher().search(percolatorQuery, MultiCollector.wrap(collector, aggregatorCollector));
-            if (aggregatorCollector != null) {
-                aggregatorCollector.postCollection();
-                aggregationPhase.execute(context);
-                        }
-            return new PercolateShardResponse(new TopDocs(collector.getTotalHits(), Lucene.EMPTY_SCORE_DOCS, 0f), Collections.emptyMap(), Collections.emptyMap(), context);
-        } else {
-            int size = context.size();
-            if (size > context.searcher().getIndexReader().maxDoc()) {
-                // prevent easy OOM if more than the total number of docs that
-                // exist is requested...
-                size = context.searcher().getIndexReader().maxDoc();
-            }
-            TopScoreDocCollector collector = TopScoreDocCollector.create(size);
-            context.searcher().search(percolatorQuery, MultiCollector.wrap(collector, aggregatorCollector));
-            if (aggregatorCollector != null) {
-                aggregatorCollector.postCollection();
-                aggregationPhase.execute(context);
-            }
-
-            TopDocs topDocs = collector.topDocs();
-            Map<Integer, String> ids = new HashMap<>(topDocs.scoreDocs.length);
-            Map<Integer, Map<String, HighlightField>> hls = new HashMap<>(topDocs.scoreDocs.length);
-            for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
-                if (context.trackScores() == false) {
-                    // No sort or tracking scores was provided, so use special
-                    // value to indicate to not show the scores:
-                    scoreDoc.score = NO_SCORE;
-                }
-
-                int segmentIdx = ReaderUtil.subIndex(scoreDoc.doc, context.searcher().getIndexReader().leaves());
-                LeafReaderContext atomicReaderContext = context.searcher().getIndexReader().leaves().get(segmentIdx);
-                final int segmentDocId = scoreDoc.doc - atomicReaderContext.docBase;
-                SingleFieldsVisitor fieldsVisitor = new SingleFieldsVisitor(UidFieldMapper.NAME);
-                atomicReaderContext.reader().document(segmentDocId, fieldsVisitor);
-                String id = fieldsVisitor.uid().id();
-                ids.put(scoreDoc.doc, id);
-                if (context.highlight() != null) {
-                    Query query = queriesRegistry.getPercolateQueries().get(new BytesRef(id));
-                    context.parsedQuery(new ParsedQuery(query));
-                    context.hitContext().cache().clear();
-                                highlightPhase.hitExecute(context, context.hitContext());
-                    hls.put(scoreDoc.doc, context.hitContext().hit().getHighlightFields());
-                }
-            }
-            return new PercolateShardResponse(topDocs, ids, hls, context);
-        }
-    }
-
-    @Override
-    public void close() {
-        cache.close();
-        }
-
-    private InternalAggregations reduceAggregations(List<PercolateShardResponse> shardResults) {
-        if (shardResults.get(0).aggregations() == null) {
-            return null;
-        }
-
-        List<InternalAggregations> aggregationsList = new ArrayList<>(shardResults.size());
-        for (PercolateShardResponse shardResult : shardResults) {
-            aggregationsList.add(shardResult.aggregations());
-        }
-        InternalAggregations aggregations = InternalAggregations.reduce(aggregationsList, new InternalAggregation.ReduceContext(bigArrays, scriptService));
-        if (aggregations != null) {
-            List<SiblingPipelineAggregator> pipelineAggregators = shardResults.get(0).pipelineAggregators();
-            if (pipelineAggregators != null) {
-                List<InternalAggregation> newAggs = StreamSupport.stream(aggregations.spliterator(), false).map((p) -> {
-                    return (InternalAggregation) p;
-                }).collect(Collectors.toList());
-                for (SiblingPipelineAggregator pipelineAggregator : pipelineAggregators) {
-                    InternalAggregation newAgg = pipelineAggregator.doReduce(new InternalAggregations(newAggs),
-                            new InternalAggregation.ReduceContext(bigArrays, scriptService));
-                    newAggs.add(newAgg);
-                }
-                aggregations = new InternalAggregations(newAggs);
-            }
-        }
-        return aggregations;
-        }
-
-    public final static class ReduceResult {
-
-        private final long count;
-        private final PercolateResponse.Match[] matches;
-        private final InternalAggregations reducedAggregations;
-
-        ReduceResult(long count, PercolateResponse.Match[] matches, InternalAggregations reducedAggregations) {
-            this.count = count;
-            this.matches = matches;
-            this.reducedAggregations = reducedAggregations;
-        }
-
-        public ReduceResult(long count, InternalAggregations reducedAggregations) {
-            this.count = count;
-            this.matches = null;
-            this.reducedAggregations = reducedAggregations;
-        }
-
-        public long count() {
-            return count;
-        }
-
-        public PercolateResponse.Match[] matches() {
-            return matches;
-        }
-
-        public InternalAggregations reducedAggregations() {
-            return reducedAggregations;
-        }
-    }
-
-
-}
diff --git a/core/src/main/java/org/elasticsearch/percolator/SingleDocumentPercolatorIndex.java b/core/src/main/java/org/elasticsearch/percolator/SingleDocumentPercolatorIndex.java
deleted file mode 100644
index 1d5268e3794..00000000000
--- a/core/src/main/java/org/elasticsearch/percolator/SingleDocumentPercolatorIndex.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.elasticsearch.percolator;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.memory.MemoryIndex;
-import org.apache.lucene.util.CloseableThreadLocal;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.internal.UidFieldMapper;
-
-import java.io.IOException;
-
-/**
- * Implementation of {@link PercolatorIndex} that can only hold a single Lucene document
- * and is optimized for that
- */
-class SingleDocumentPercolatorIndex implements PercolatorIndex {
-
-    private final CloseableThreadLocal<MemoryIndex> cache;
-
-    SingleDocumentPercolatorIndex(CloseableThreadLocal<MemoryIndex> cache) {
-        this.cache = cache;
-    }
-
-    @Override
-    public void prepare(PercolateContext context, ParsedDocument parsedDocument) {
-        MemoryIndex memoryIndex = cache.get();
-        for (IndexableField field : parsedDocument.rootDoc().getFields()) {
-            Analyzer analyzer = context.analysisService().defaultIndexAnalyzer();
-            DocumentMapper documentMapper = context.mapperService().documentMapper(parsedDocument.type());
-            if (documentMapper != null && documentMapper.mappers().getMapper(field.name()) != null) {
-                analyzer =  documentMapper.mappers().indexAnalyzer();
-            }
-            if (field.fieldType().indexOptions() == IndexOptions.NONE && field.name().equals(UidFieldMapper.NAME)) {
-                continue;
-            }
-            try {
-                // TODO: instead of passing null here, we can have a CTL<Map<String,TokenStream>> and pass previous,
-                // like the indexer does
-                try (TokenStream tokenStream = field.tokenStream(analyzer, null)) {
-                    if (tokenStream != null) {
-                        memoryIndex.addField(field.name(), tokenStream, field.boost());
-                    }
-                 }
-            } catch (Exception e) {
-                throw new ElasticsearchException("Failed to create token stream for [" + field.name() + "]", e);
-            }
-        }
-        context.initialize(new DocEngineSearcher(memoryIndex), parsedDocument);
-    }
-
-    private class DocEngineSearcher extends Engine.Searcher {
-
-        private final MemoryIndex memoryIndex;
-
-        public DocEngineSearcher(MemoryIndex memoryIndex) {
-            super("percolate", memoryIndex.createSearcher());
-            this.memoryIndex = memoryIndex;
-        }
-
-        @Override
-        public void close() {
-            try {
-                this.reader().close();
-                memoryIndex.reset();
-            } catch (IOException e) {
-                throw new ElasticsearchException("failed to close percolator in-memory index", e);
-            }
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/plugins/DummyPluginInfo.java b/core/src/main/java/org/elasticsearch/plugins/DummyPluginInfo.java
index a7d088ce214..5dab19581a3 100644
--- a/core/src/main/java/org/elasticsearch/plugins/DummyPluginInfo.java
+++ b/core/src/main/java/org/elasticsearch/plugins/DummyPluginInfo.java
@@ -20,9 +20,9 @@ package org.elasticsearch.plugins;
 
 public class DummyPluginInfo extends PluginInfo {
 
-    private DummyPluginInfo(String name, String description, String version, String classname, boolean isolated) {
-        super(name, description, version, classname, isolated);
+    private DummyPluginInfo(String name, String description, String version, String classname) {
+        super(name, description, version, classname);
     }
 
-    public static final DummyPluginInfo INSTANCE = new DummyPluginInfo("dummy_plugin_name", "dummy plugin description", "dummy_plugin_version", "DummyPluginName", true);
+    public static final DummyPluginInfo INSTANCE = new DummyPluginInfo("dummy_plugin_name", "dummy plugin description", "dummy_plugin_version", "DummyPluginName");
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index 767f6d42179..4096ad57885 100644
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -19,16 +19,18 @@
 
 package org.elasticsearch.plugins;
 
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Build;
 import org.elasticsearch.Version;
 import org.elasticsearch.bootstrap.JarHell;
-import org.elasticsearch.common.cli.CliTool;
-import org.elasticsearch.common.cli.Terminal;
-import org.elasticsearch.common.cli.UserError;
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserError;
 import org.elasticsearch.common.hash.MessageDigests;
 import org.elasticsearch.common.io.FileSystemUtils;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 
 import java.io.BufferedReader;
@@ -44,18 +46,22 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFileAttributes;
 import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Objects;
 import java.util.Set;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
 import static java.util.Collections.unmodifiableSet;
-import static org.elasticsearch.common.cli.Terminal.Verbosity.VERBOSE;
+import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
 
 /**
@@ -63,9 +69,9 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet;
  *
  * The install command takes a plugin id, which may be any of the following:
  * <ul>
- *     <li>An official elasticsearch plugin name</li>
- *     <li>Maven coordinates to a plugin zip</li>
- *     <li>A URL to a plugin zip</li>
+ * <li>An official elasticsearch plugin name</li>
+ * <li>Maven coordinates to a plugin zip</li>
+ * <li>A URL to a plugin zip</li>
  * </ul>
  *
  * Plugins are packaged as zip files. Each packaged plugin must contain a
@@ -74,9 +80,9 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet;
  * The installation process first extracts the plugin files into a temporary
  * directory in order to verify the plugin satisfies the following requirements:
  * <ul>
- *     <li>Jar hell does not exist, either between the plugin's own jars, or with elasticsearch</li>
- *     <li>The plugin is not a module already provided with elasticsearch</li>
- *     <li>If the plugin contains extra security permissions, the policy file is validated</li>
+ * <li>Jar hell does not exist, either between the plugin's own jars, or with elasticsearch</li>
+ * <li>The plugin is not a module already provided with elasticsearch</li>
+ * <li>If the plugin contains extra security permissions, the policy file is validated</li>
  * </ul>
  * <p>
  * A plugin may also contain an optional {@code bin} directory which contains scripts. The
@@ -88,48 +94,77 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet;
  * elasticsearch config directory, using the name of the plugin. If any files to be installed
  * already exist, they will be skipped.
  */
-class InstallPluginCommand extends CliTool.Command {
+class InstallPluginCommand extends Command {
 
     private static final String PROPERTY_SUPPORT_STAGING_URLS = "es.plugins.staging";
 
     // TODO: make this a resource file generated by gradle
     static final Set<String> MODULES = unmodifiableSet(newHashSet(
-        "lang-expression",
-        "lang-groovy"));
+            "ingest-grok",
+            "lang-expression",
+            "lang-groovy",
+            "lang-painless",
+            "reindex"));
 
     // TODO: make this a resource file generated by gradle
-    static final Set<String> OFFICIAL_PLUGINS = unmodifiableSet(newHashSet(
-        "analysis-icu",
-        "analysis-kuromoji",
-        "analysis-phonetic",
-        "analysis-smartcn",
-        "analysis-stempel",
-        "delete-by-query",
-        "discovery-azure",
-        "discovery-ec2",
-        "discovery-gce",
-        "lang-javascript",
-        "lang-painless",
-        "lang-python",
-        "mapper-attachments",
-        "mapper-murmur3",
-        "mapper-size",
-        "repository-azure",
-        "repository-hdfs",
-        "repository-s3",
-        "store-smb"));
+    static final Set<String> OFFICIAL_PLUGINS = unmodifiableSet(new LinkedHashSet<>(Arrays.asList(
+            "analysis-icu",
+            "analysis-kuromoji",
+            "analysis-phonetic",
+            "analysis-smartcn",
+            "analysis-stempel",
+            "delete-by-query",
+            "discovery-azure",
+            "discovery-ec2",
+            "discovery-gce",
+            "ingest-attachment",
+            "ingest-geoip",
+            "lang-javascript",
+            "lang-python",
+            "mapper-attachments",
+            "mapper-murmur3",
+            "mapper-size",
+            "repository-azure",
+            "repository-hdfs",
+            "repository-s3",
+            "store-smb",
+            "xpack")));
 
-    private final String pluginId;
-    private final boolean batch;
+    private final Environment env;
+    private final OptionSpec<Void> batchOption;
+    private final OptionSpec<String> arguments;
 
-    InstallPluginCommand(Terminal terminal, String pluginId, boolean batch) {
-        super(terminal);
-        this.pluginId = pluginId;
-        this.batch = batch;
+    InstallPluginCommand(Environment env) {
+        super("Install a plugin");
+        this.env = env;
+        this.batchOption = parser.acceptsAll(Arrays.asList("b", "batch"),
+                "Enable batch mode explicitly, automatic confirmation of security permission");
+        this.arguments = parser.nonOptions("plugin id");
     }
 
     @Override
-    public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
+    protected void printAdditionalHelp(Terminal terminal) {
+        terminal.println("The following official plugins may be installed by name:");
+        for (String plugin : OFFICIAL_PLUGINS) {
+            terminal.println("  " + plugin);
+        }
+        terminal.println("");
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options) throws Exception {
+        // TODO: in jopt-simple 5.0 we can enforce a min/max number of positional args
+        List<String> args = arguments.values(options);
+        if (args.size() != 1) {
+            throw new UserError(ExitCodes.USAGE, "Must supply a single plugin id argument");
+        }
+        String pluginId = args.get(0);
+        boolean isBatch = options.has(batchOption) || System.console() == null;
+        execute(terminal, pluginId, isBatch);
+    }
+
+    // pkg private for testing
+    void execute(Terminal terminal, String pluginId, boolean isBatch) throws Exception {
 
         // TODO: remove this leniency!! is it needed anymore?
         if (Files.exists(env.pluginsFile()) == false) {
@@ -137,24 +172,29 @@ class InstallPluginCommand extends CliTool.Command {
             Files.createDirectory(env.pluginsFile());
         }
 
-        Path pluginZip = download(pluginId, env.tmpFile());
+        Path pluginZip = download(terminal, pluginId, env.tmpFile());
         Path extractedZip = unzip(pluginZip, env.pluginsFile());
-        install(extractedZip, env);
-
-        return CliTool.ExitStatus.OK;
+        install(terminal, isBatch, extractedZip);
     }
 
     /** Downloads the plugin and returns the file it was downloaded to. */
-    private Path download(String pluginId, Path tmpDir) throws Exception {
+    private Path download(Terminal terminal, String pluginId, Path tmpDir) throws Exception {
         if (OFFICIAL_PLUGINS.contains(pluginId)) {
             final String version = Version.CURRENT.toString();
             final String url;
             if (System.getProperty(PROPERTY_SUPPORT_STAGING_URLS, "false").equals("true")) {
-                url = String.format(Locale.ROOT, "https://download.elastic.co/elasticsearch/staging/%1$s-%2$s/org/elasticsearch/plugin/%3$s/%1$s/%3$s-%1$s.zip",
-                    version, Build.CURRENT.shortHash(), pluginId);
+                url = String.format(
+                        Locale.ROOT,
+                        "https://download.elastic.co/elasticsearch/staging/%1$s-%2$s/org/elasticsearch/plugin/%3$s/%1$s/%3$s-%1$s.zip",
+                        version,
+                        Build.CURRENT.shortHash(),
+                        pluginId);
             } else {
-                url = String.format(Locale.ROOT, "https://download.elastic.co/elasticsearch/release/org/elasticsearch/plugin/%1$s/%2$s/%1$s-%2$s.zip",
-                    pluginId, version);
+                url = String.format(
+                        Locale.ROOT,
+                        "https://download.elastic.co/elasticsearch/release/org/elasticsearch/plugin/%1$s/%2$s/%1$s-%2$s.zip",
+                        pluginId,
+                        version);
             }
             terminal.println("-> Downloading " + pluginId + " from elastic");
             return downloadZipAndChecksum(url, tmpDir);
@@ -164,7 +204,7 @@ class InstallPluginCommand extends CliTool.Command {
         String[] coordinates = pluginId.split(":");
         if (coordinates.length == 3 && pluginId.contains("/") == false) {
             String mavenUrl = String.format(Locale.ROOT, "https://repo1.maven.org/maven2/%1$s/%2$s/%3$s/%2$s-%3$s.zip",
-                coordinates[0].replace(".", "/") /* groupId */, coordinates[1] /* artifactId */, coordinates[2] /* version */);
+                    coordinates[0].replace(".", "/") /* groupId */, coordinates[1] /* artifactId */, coordinates[2] /* version */);
             terminal.println("-> Downloading " + pluginId + " from maven central");
             return downloadZipAndChecksum(mavenUrl, tmpDir);
         }
@@ -195,14 +235,14 @@ class InstallPluginCommand extends CliTool.Command {
             BufferedReader checksumReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
             expectedChecksum = checksumReader.readLine();
             if (checksumReader.readLine() != null) {
-                throw new UserError(CliTool.ExitStatus.IO_ERROR, "Invalid checksum file at " + checksumUrl);
+                throw new UserError(ExitCodes.IO_ERROR, "Invalid checksum file at " + checksumUrl);
             }
         }
 
         byte[] zipbytes = Files.readAllBytes(zip);
         String gotChecksum = MessageDigests.toHexString(MessageDigests.sha1().digest(zipbytes));
         if (expectedChecksum.equals(gotChecksum) == false) {
-            throw new UserError(CliTool.ExitStatus.IO_ERROR, "SHA1 mismatch, expected " + expectedChecksum + " but got " + gotChecksum);
+            throw new UserError(ExitCodes.IO_ERROR, "SHA1 mismatch, expected " + expectedChecksum + " but got " + gotChecksum);
         }
 
         return zip;
@@ -210,8 +250,8 @@ class InstallPluginCommand extends CliTool.Command {
 
     private Path unzip(Path zip, Path pluginsDir) throws IOException, UserError {
         // unzip plugin to a staging temp dir
-        Path target = Files.createTempDirectory(pluginsDir, ".installing-");
-        Files.createDirectories(target);
+
+        final Path target = stagingDirectory(pluginsDir);
 
         boolean hasEsDir = false;
         // TODO: we should wrap this in a try/catch and try deleting the target dir on failure?
@@ -225,7 +265,14 @@ class InstallPluginCommand extends CliTool.Command {
                 }
                 hasEsDir = true;
                 Path targetFile = target.resolve(entry.getName().substring("elasticsearch/".length()));
-                // TODO: handle name being an absolute path
+
+                // Using the entry name as a path can result in an entry outside of the plugin dir, either if the
+                // name starts with the root of the filesystem, or it is a relative entry like ../whatever.
+                // This check attempts to identify both cases by first normalizing the path (which removes foo/..)
+                // and ensuring the normalized entry is still rooted with the target plugin directory.
+                if (targetFile.normalize().startsWith(target) == false) {
+                    throw new IOException("Zip contains entry name '" + entry.getName() + "' resolving outside of plugin directory");
+                }
 
                 // be on the safe side: do not rely on that directories are always extracted
                 // before their children (although this makes sense, but is it guaranteed?)
@@ -233,7 +280,7 @@ class InstallPluginCommand extends CliTool.Command {
                 if (entry.isDirectory() == false) {
                     try (OutputStream out = Files.newOutputStream(targetFile)) {
                         int len;
-                        while((len = zipInput.read(buffer)) >= 0) {
+                        while ((len = zipInput.read(buffer)) >= 0) {
                             out.write(buffer, 0, len);
                         }
                     }
@@ -244,13 +291,46 @@ class InstallPluginCommand extends CliTool.Command {
         Files.delete(zip);
         if (hasEsDir == false) {
             IOUtils.rm(target);
-            throw new UserError(CliTool.ExitStatus.DATA_ERROR, "`elasticsearch` directory is missing in the plugin zip");
+            throw new UserError(ExitCodes.DATA_ERROR, "`elasticsearch` directory is missing in the plugin zip");
         }
         return target;
     }
 
+    private Path stagingDirectory(Path pluginsDir) throws IOException {
+        try {
+            Set<PosixFilePermission> perms = new HashSet<>();
+            perms.add(PosixFilePermission.OWNER_EXECUTE);
+            perms.add(PosixFilePermission.OWNER_READ);
+            perms.add(PosixFilePermission.OWNER_WRITE);
+            perms.add(PosixFilePermission.GROUP_READ);
+            perms.add(PosixFilePermission.GROUP_EXECUTE);
+            perms.add(PosixFilePermission.OTHERS_READ);
+            perms.add(PosixFilePermission.OTHERS_EXECUTE);
+            return Files.createTempDirectory(pluginsDir, ".installing-", PosixFilePermissions.asFileAttribute(perms));
+        } catch (IllegalArgumentException e) {
+            // Jimfs throws an IAE where it should throw an UOE
+            // remove when google/jimfs#30 is integrated into Jimfs
+            // and the Jimfs test dependency is upgraded to include
+            // this pull request
+            final StackTraceElement[] elements = e.getStackTrace();
+            if (elements.length >= 1 &&
+                elements[0].getClassName().equals("com.google.common.jimfs.AttributeService") &&
+                elements[0].getMethodName().equals("setAttributeInternal")) {
+                return stagingDirectoryWithoutPosixPermissions(pluginsDir);
+            } else {
+                throw e;
+            }
+        } catch (UnsupportedOperationException e) {
+            return stagingDirectoryWithoutPosixPermissions(pluginsDir);
+        }
+    }
+
+    private Path stagingDirectoryWithoutPosixPermissions(Path pluginsDir) throws IOException {
+        return Files.createTempDirectory(pluginsDir, ".installing-");
+    }
+
     /** Load information about the plugin, and verify it can be installed with no errors. */
-    private PluginInfo verify(Path pluginRoot, Environment env) throws Exception {
+    private PluginInfo verify(Terminal terminal, Path pluginRoot, boolean isBatch) throws Exception {
         // read and validate the plugin descriptor
         PluginInfo info = PluginInfo.readFromProperties(pluginRoot);
         terminal.println(VERBOSE, info.toString());
@@ -258,36 +338,30 @@ class InstallPluginCommand extends CliTool.Command {
         // don't let luser install plugin as a module...
         // they might be unavoidably in maven central and are packaged up the same way)
         if (MODULES.contains(info.getName())) {
-            throw new UserError(CliTool.ExitStatus.USAGE, "plugin '" + info.getName() + "' cannot be installed like this, it is a system module");
+            throw new UserError(ExitCodes.USAGE, "plugin '" + info.getName() + "' cannot be installed like this, it is a system module");
         }
 
         // check for jar hell before any copying
-        jarHellCheck(pluginRoot, env.pluginsFile(), info.isIsolated());
+        jarHellCheck(pluginRoot, env.pluginsFile());
 
         // read optional security policy (extra permissions)
         // if it exists, confirm or warn the user
         Path policy = pluginRoot.resolve(PluginInfo.ES_PLUGIN_POLICY);
         if (Files.exists(policy)) {
-            PluginSecurity.readPolicy(policy, terminal, env, batch);
+            PluginSecurity.readPolicy(policy, terminal, env, isBatch);
         }
 
         return info;
     }
 
     /** check a candidate plugin for jar hell before installing it */
-    private void jarHellCheck(Path candidate, Path pluginsDir, boolean isolated) throws Exception {
+    void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {
         // create list of current jars in classpath
         final List<URL> jars = new ArrayList<>();
         jars.addAll(Arrays.asList(JarHell.parseClassPath()));
 
         // read existing bundles. this does some checks on the installation too.
-        List<PluginsService.Bundle> bundles = PluginsService.getPluginBundles(pluginsDir);
-
-        // if we aren't isolated, we need to jarhellcheck against any other non-isolated plugins
-        // that's always the first bundle
-        if (isolated == false) {
-            jars.addAll(bundles.get(0).urls);
-        }
+        PluginsService.getPluginBundles(pluginsDir);
 
         // add plugin jars to the list
         Path pluginJars[] = FileSystemUtils.files(candidate, "*.jar");
@@ -305,16 +379,19 @@ class InstallPluginCommand extends CliTool.Command {
      * Installs the plugin from {@code tmpRoot} into the plugins dir.
      * If the plugin has a bin dir and/or a config dir, those are copied.
      */
-    private void install(Path tmpRoot, Environment env) throws Exception {
+    private void install(Terminal terminal, boolean isBatch, Path tmpRoot) throws Exception {
         List<Path> deleteOnFailure = new ArrayList<>();
         deleteOnFailure.add(tmpRoot);
 
         try {
-            PluginInfo info = verify(tmpRoot, env);
+            PluginInfo info = verify(terminal, tmpRoot, isBatch);
 
             final Path destination = env.pluginsFile().resolve(info.getName());
             if (Files.exists(destination)) {
-                throw new UserError(CliTool.ExitStatus.USAGE, "plugin directory " + destination.toAbsolutePath() + " already exists. To update the plugin, uninstall it first using 'remove " + info.getName() + "' command");
+                throw new UserError(
+                    ExitCodes.USAGE,
+                    "plugin directory " + destination.toAbsolutePath() +
+                        " already exists. To update the plugin, uninstall it first using 'remove " + info.getName() + "' command");
             }
 
             Path tmpBinDir = tmpRoot.resolve("bin");
@@ -347,32 +424,34 @@ class InstallPluginCommand extends CliTool.Command {
     /** Copies the files from {@code tmpBinDir} into {@code destBinDir}, along with permissions from dest dirs parent. */
     private void installBin(PluginInfo info, Path tmpBinDir, Path destBinDir) throws Exception {
         if (Files.isDirectory(tmpBinDir) == false) {
-            throw new UserError(CliTool.ExitStatus.IO_ERROR, "bin in plugin " + info.getName() + " is not a directory");
+            throw new UserError(ExitCodes.IO_ERROR, "bin in plugin " + info.getName() + " is not a directory");
         }
         Files.createDirectory(destBinDir);
 
         // setup file attributes for the installed files to those of the parent dir
-        Set<PosixFilePermission> perms = new HashSet<>();
-        PosixFileAttributeView binAttrs = Files.getFileAttributeView(destBinDir.getParent(), PosixFileAttributeView.class);
-        if (binAttrs != null) {
-            perms = new HashSet<>(binAttrs.readAttributes().permissions());
+        final Set<PosixFilePermission> perms = new HashSet<>();
+        final PosixFileAttributeView binAttributeView = Files.getFileAttributeView(destBinDir.getParent(), PosixFileAttributeView.class);
+        if (binAttributeView != null) {
+            perms.addAll(binAttributeView.readAttributes().permissions());
             // setting execute bits, since this just means "the file is executable", and actual execution requires read
             perms.add(PosixFilePermission.OWNER_EXECUTE);
             perms.add(PosixFilePermission.GROUP_EXECUTE);
             perms.add(PosixFilePermission.OTHERS_EXECUTE);
         }
 
-        try (DirectoryStream<Path> stream  = Files.newDirectoryStream(tmpBinDir)) {
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(tmpBinDir)) {
             for (Path srcFile : stream) {
                 if (Files.isDirectory(srcFile)) {
-                    throw new UserError(CliTool.ExitStatus.DATA_ERROR, "Directories not allowed in bin dir for plugin " + info.getName() + ", found " + srcFile.getFileName());
+                    throw new UserError(
+                        ExitCodes.DATA_ERROR,
+                        "Directories not allowed in bin dir for plugin " + info.getName() + ", found " + srcFile.getFileName());
                 }
 
                 Path destFile = destBinDir.resolve(tmpBinDir.relativize(srcFile));
                 Files.copy(srcFile, destFile);
 
-                if (perms.isEmpty() == false) {
-                    PosixFileAttributeView view = Files.getFileAttributeView(destFile, PosixFileAttributeView.class);
+                final PosixFileAttributeView view = Files.getFileAttributeView(destFile, PosixFileAttributeView.class);
+                if (view != null) {
                     view.setPermissions(perms);
                 }
             }
@@ -386,24 +465,43 @@ class InstallPluginCommand extends CliTool.Command {
      */
     private void installConfig(PluginInfo info, Path tmpConfigDir, Path destConfigDir) throws Exception {
         if (Files.isDirectory(tmpConfigDir) == false) {
-            throw new UserError(CliTool.ExitStatus.IO_ERROR, "config in plugin " + info.getName() + " is not a directory");
+            throw new UserError(ExitCodes.IO_ERROR, "config in plugin " + info.getName() + " is not a directory");
         }
 
         // create the plugin's config dir "if necessary"
         Files.createDirectories(destConfigDir);
+        final PosixFileAttributeView destConfigDirAttributesView =
+            Files.getFileAttributeView(destConfigDir.getParent(), PosixFileAttributeView.class);
+        final PosixFileAttributes destConfigDirAttributes =
+            destConfigDirAttributesView != null ? destConfigDirAttributesView.readAttributes() : null;
+        if (destConfigDirAttributes != null) {
+            setOwnerGroup(destConfigDir, destConfigDirAttributes);
+        }
 
-        try (DirectoryStream<Path> stream  = Files.newDirectoryStream(tmpConfigDir)) {
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(tmpConfigDir)) {
             for (Path srcFile : stream) {
                 if (Files.isDirectory(srcFile)) {
-                    throw new UserError(CliTool.ExitStatus.DATA_ERROR, "Directories not allowed in config dir for plugin " + info.getName());
+                    throw new UserError(ExitCodes.DATA_ERROR, "Directories not allowed in config dir for plugin " + info.getName());
                 }
 
                 Path destFile = destConfigDir.resolve(tmpConfigDir.relativize(srcFile));
                 if (Files.exists(destFile) == false) {
                     Files.copy(srcFile, destFile);
+                    if (destConfigDirAttributes != null) {
+                        setOwnerGroup(destFile, destConfigDirAttributes);
+                    }
                 }
             }
         }
         IOUtils.rm(tmpConfigDir); // clean up what we just copied
     }
+
+    private static void setOwnerGroup(final Path path, final PosixFileAttributes attributes) throws IOException {
+        Objects.requireNonNull(attributes);
+        PosixFileAttributeView fileAttributeView = Files.getFileAttributeView(path, PosixFileAttributeView.class);
+        assert fileAttributeView != null;
+        fileAttributeView.setOwner(attributes.owner());
+        fileAttributeView.setGroup(attributes.group());
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java b/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
index 6abed4e6bc2..953e698a4c2 100644
--- a/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
@@ -24,22 +24,25 @@ import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
-import org.elasticsearch.common.cli.CliTool;
-import org.elasticsearch.common.cli.Terminal;
-import org.elasticsearch.common.settings.Settings;
+import joptsimple.OptionSet;
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.env.Environment;
 
 /**
  * A command for the plugin cli to list plugins installed in elasticsearch.
  */
-class ListPluginsCommand extends CliTool.Command {
+class ListPluginsCommand extends Command {
 
-    ListPluginsCommand(Terminal terminal) {
-        super(terminal);
+    private final Environment env;
+
+    ListPluginsCommand(Environment env) {
+        super("Lists installed elasticsearch plugins");
+        this.env = env;
     }
 
     @Override
-    public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
+    protected void execute(Terminal terminal, OptionSet options) throws Exception {
         if (Files.exists(env.pluginsFile()) == false) {
             throw new IOException("Plugins directory missing: " + env.pluginsFile());
         }
@@ -50,7 +53,5 @@ class ListPluginsCommand extends CliTool.Command {
                 terminal.println(plugin.getFileName().toString());
             }
         }
-
-        return CliTool.ExitStatus.OK;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginCli.java b/core/src/main/java/org/elasticsearch/plugins/PluginCli.java
index df402e6359d..be06ea7db1c 100644
--- a/core/src/main/java/org/elasticsearch/plugins/PluginCli.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginCli.java
@@ -19,41 +19,24 @@
 
 package org.elasticsearch.plugins;
 
-import org.apache.commons.cli.CommandLine;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.cli.CliTool;
-import org.elasticsearch.common.cli.CliToolConfig;
-import org.elasticsearch.common.cli.Terminal;
+import org.elasticsearch.cli.MultiCommand;
+import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 
-import java.util.Locale;
-
-import static org.elasticsearch.common.cli.CliToolConfig.Builder.cmd;
-import static org.elasticsearch.common.cli.CliToolConfig.Builder.option;
-
 /**
  * A cli tool for adding, removing and listing plugins for elasticsearch.
  */
-public class PluginCli extends CliTool {
+public class PluginCli extends MultiCommand {
 
-    // commands
-    private static final String LIST_CMD_NAME = "list";
-    private static final String INSTALL_CMD_NAME = "install";
-    private static final String REMOVE_CMD_NAME = "remove";
-
-    // usage config
-    private static final CliToolConfig.Cmd LIST_CMD = cmd(LIST_CMD_NAME, ListPluginsCommand.class).build();
-    private static final CliToolConfig.Cmd INSTALL_CMD = cmd(INSTALL_CMD_NAME, InstallPluginCommand.class)
-        .options(option("b", "batch").required(false))
-        .build();
-    private static final CliToolConfig.Cmd REMOVE_CMD = cmd(REMOVE_CMD_NAME, RemovePluginCommand.class).build();
-
-    static final CliToolConfig CONFIG = CliToolConfig.config("plugin", PluginCli.class)
-            .cmds(LIST_CMD, INSTALL_CMD, REMOVE_CMD)
-            .build();
+    public PluginCli(Environment env) {
+        super("A tool for managing installed elasticsearch plugins");
+        subcommands.put("list", new ListPluginsCommand(env));
+        subcommands.put("install", new InstallPluginCommand(env));
+        subcommands.put("remove", new RemovePluginCommand(env));
+    }
 
     public static void main(String[] args) throws Exception {
         // initialize default for es.logger.level because we will not read the logging.yml
@@ -64,61 +47,13 @@ public class PluginCli extends CliTool {
         // executed there is no way of knowing where the logfiles should be placed. For example, if elasticsearch
         // is run as service then the logs should be at /var/log/elasticsearch but when started from the tar they should be at es.home/logs.
         // Therefore we print to Terminal.
-        Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.builder()
+        Environment loggingEnvironment = InternalSettingsPreparer.prepareEnvironment(Settings.builder()
                 .put("appender.terminal.type", "terminal")
                 .put("rootLogger", "${es.logger.level}, terminal")
                 .put("es.logger.level", loggerLevel)
                 .build(), Terminal.DEFAULT);
-        // configure but do not read the logging conf file
-        LogConfigurator.configure(env.settings(), false);
-        int status = new PluginCli(Terminal.DEFAULT).execute(args).status();
-        exit(status);
-    }
-
-    @SuppressForbidden(reason = "Allowed to exit explicitly from #main()")
-    private static void exit(int status) {
-        System.exit(status);
-    }
-
-    PluginCli(Terminal terminal) {
-        super(CONFIG, terminal);
-    }
-
-    @Override
-    protected Command parse(String cmdName, CommandLine cli) throws Exception {
-        switch (cmdName.toLowerCase(Locale.ROOT)) {
-            case LIST_CMD_NAME:
-                return new ListPluginsCommand(terminal);
-            case INSTALL_CMD_NAME:
-                return parseInstallPluginCommand(cli);
-            case REMOVE_CMD_NAME:
-                return parseRemovePluginCommand(cli);
-            default:
-                assert false : "can't get here as cmd name is validated before this method is called";
-                return exitCmd(ExitStatus.USAGE);
-        }
-    }
-
-    private Command parseInstallPluginCommand(CommandLine cli) {
-        String[] args = cli.getArgs();
-        if (args.length != 1) {
-            return exitCmd(ExitStatus.USAGE, terminal, "Must supply a single plugin id argument");
-        }
-
-        boolean batch = System.console() == null;
-        if (cli.hasOption("b")) {
-            batch = true;
-        }
-
-        return new InstallPluginCommand(terminal, args[0], batch);
-    }
-
-    private Command parseRemovePluginCommand(CommandLine cli) {
-        String[] args = cli.getArgs();
-        if (args.length != 1) {
-            return exitCmd(ExitStatus.USAGE, terminal, "Must supply a single plugin name argument");
-        }
-
-        return new RemovePluginCommand(terminal, args[0]);
+        LogConfigurator.configure(loggingEnvironment.settings(), false);
+        Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, Terminal.DEFAULT);
+        exit(new PluginCli(env).main(args, Terminal.DEFAULT));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java b/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java
index 73464d054dd..bd063b3312c 100644
--- a/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java
@@ -44,14 +44,12 @@ public class PluginInfo implements Streamable, ToXContent {
         static final XContentBuilderString URL = new XContentBuilderString("url");
         static final XContentBuilderString VERSION = new XContentBuilderString("version");
         static final XContentBuilderString CLASSNAME = new XContentBuilderString("classname");
-        static final XContentBuilderString ISOLATED = new XContentBuilderString("isolated");
     }
 
     private String name;
     private String description;
     private String version;
     private String classname;
-    private boolean isolated;
 
     public PluginInfo() {
     }
@@ -63,12 +61,11 @@ public class PluginInfo implements Streamable, ToXContent {
      * @param description Its description
      * @param version     Version number
      */
-    PluginInfo(String name, String description, String version, String classname, boolean isolated) {
+    PluginInfo(String name, String description, String version, String classname) {
         this.name = name;
         this.description = description;
         this.version = version;
         this.classname = classname;
-        this.isolated = isolated;
     }
 
     /** reads (and validates) plugin metadata descriptor file */
@@ -106,13 +103,12 @@ public class PluginInfo implements Streamable, ToXContent {
         }
         JarHell.checkVersionFormat(javaVersionString);
         JarHell.checkJavaVersion(name, javaVersionString);
-        boolean isolated = Boolean.parseBoolean(props.getProperty("isolated", "true"));
         String classname = props.getProperty("classname");
         if (classname == null) {
             throw new IllegalArgumentException("Property [classname] is missing for plugin [" + name + "]");
         }
 
-        return new PluginInfo(name, description, version, classname, isolated);
+        return new PluginInfo(name, description, version, classname);
     }
 
     /**
@@ -129,13 +125,6 @@ public class PluginInfo implements Streamable, ToXContent {
         return description;
     }
 
-    /**
-     * @return true if plugin has isolated classloader
-     */
-    public boolean isIsolated() {
-        return isolated;
-    }
-
     /**
      * @return plugin's classname
      */
@@ -162,7 +151,6 @@ public class PluginInfo implements Streamable, ToXContent {
         this.description = in.readString();
         this.version = in.readString();
         this.classname = in.readString();
-        this.isolated = in.readBoolean();
     }
 
     @Override
@@ -171,7 +159,6 @@ public class PluginInfo implements Streamable, ToXContent {
         out.writeString(description);
         out.writeString(version);
         out.writeString(classname);
-        out.writeBoolean(isolated);
     }
 
     @Override
@@ -181,7 +168,6 @@ public class PluginInfo implements Streamable, ToXContent {
         builder.field(Fields.VERSION, version);
         builder.field(Fields.DESCRIPTION, description);
         builder.field(Fields.CLASSNAME, classname);
-        builder.field(Fields.ISOLATED, isolated);
         builder.endObject();
 
         return builder;
@@ -212,8 +198,7 @@ public class PluginInfo implements Streamable, ToXContent {
                 .append("Name: ").append(name).append("\n")
                 .append("Description: ").append(description).append("\n")
                 .append("Version: ").append(version).append("\n")
-                .append(" * Classname: ").append(classname).append("\n")
-                .append(" * Isolated: ").append(isolated);
+                .append(" * Classname: ").append(classname);
 
         return information.toString();
     }
diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginSecurity.java b/core/src/main/java/org/elasticsearch/plugins/PluginSecurity.java
index b14bcaf2ff3..f9c3d1826c9 100644
--- a/core/src/main/java/org/elasticsearch/plugins/PluginSecurity.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginSecurity.java
@@ -20,8 +20,8 @@
 package org.elasticsearch.plugins;
 
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.common.cli.Terminal;
-import org.elasticsearch.common.cli.Terminal.Verbosity;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.Terminal.Verbosity;
 import org.elasticsearch.env.Environment;
 
 import java.io.IOException;
diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
index 3e36c5d8f09..c1f3043cd32 100644
--- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
@@ -37,6 +37,7 @@ import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexModule;
 
@@ -71,7 +72,8 @@ public class PluginsService extends AbstractComponent {
      */
     private final List<Tuple<PluginInfo, Plugin>> plugins;
     private final PluginsAndModules info;
-    public static final Setting<List<String>> MANDATORY_SETTING = Setting.listSetting("plugin.mandatory", Collections.emptyList(), Function.identity(), false, Setting.Scope.CLUSTER);
+    public static final Setting<List<String>> MANDATORY_SETTING =
+        Setting.listSetting("plugin.mandatory", Collections.emptyList(), Function.identity(), Property.NodeScope);
 
     private final Map<Plugin, List<OnModuleReference>> onModuleReferences;
 
@@ -101,7 +103,7 @@ public class PluginsService extends AbstractComponent {
         // first we load plugins that are on the classpath. this is for tests and transport clients
         for (Class<? extends Plugin> pluginClass : classpathPlugins) {
             Plugin plugin = loadPlugin(pluginClass, settings);
-            PluginInfo pluginInfo = new PluginInfo(plugin.name(), plugin.description(), "NA", pluginClass.getName(), false);
+            PluginInfo pluginInfo = new PluginInfo(plugin.name(), plugin.description(), "NA", pluginClass.getName());
             if (logger.isTraceEnabled()) {
                 logger.trace("plugin loaded from classpath [{}]", pluginInfo);
             }
@@ -300,9 +302,6 @@ public class PluginsService extends AbstractComponent {
                     continue; // skip over .DS_Store etc
                 }
                 PluginInfo info = PluginInfo.readFromProperties(module);
-                if (!info.isIsolated()) {
-                    throw new IllegalStateException("modules must be isolated: " + info);
-                }
                 Bundle bundle = new Bundle();
                 bundle.plugins.add(info);
                 // gather urls for jar files
@@ -327,8 +326,6 @@ public class PluginsService extends AbstractComponent {
         }
 
         List<Bundle> bundles = new ArrayList<>();
-        // a special purgatory for plugins that directly depend on each other
-        bundles.add(new Bundle());
 
         try (DirectoryStream<Path> stream = Files.newDirectoryStream(pluginsDirectory)) {
             for (Path plugin : stream) {
@@ -352,13 +349,8 @@ public class PluginsService extends AbstractComponent {
                         urls.add(jar.toRealPath().toUri().toURL());
                     }
                 }
-                final Bundle bundle;
-                if (info.isIsolated() == false) {
-                    bundle = bundles.get(0); // purgatory
-                } else {
-                    bundle = new Bundle();
-                    bundles.add(bundle);
-                }
+                final Bundle bundle = new Bundle();
+                bundles.add(bundle);
                 bundle.plugins.add(info);
                 bundle.urls.addAll(urls);
             }
diff --git a/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
index 8ce1056bbfd..a3e6c375f83 100644
--- a/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
@@ -19,40 +19,55 @@
 
 package org.elasticsearch.plugins;
 
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.cli.CliTool;
-import org.elasticsearch.common.cli.Terminal;
-import org.elasticsearch.common.cli.UserError;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.elasticsearch.common.cli.Terminal.Verbosity.VERBOSE;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.UserError;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.env.Environment;
+
+import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
 
 /**
  * A command for the plugin cli to remove a plugin from elasticsearch.
  */
-class RemovePluginCommand extends CliTool.Command {
-    private final String pluginName;
+class RemovePluginCommand extends Command {
 
-    public RemovePluginCommand(Terminal terminal, String pluginName) {
-        super(terminal);
-        this.pluginName = pluginName;
+    private final Environment env;
+    private final OptionSpec<String> arguments;
+
+    RemovePluginCommand(Environment env) {
+        super("Removes a plugin from elasticsearch");
+        this.env = env;
+        this.arguments = parser.nonOptions("plugin name");
     }
 
     @Override
-    public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
+    protected void execute(Terminal terminal, OptionSet options) throws Exception {
+        // TODO: in jopt-simple 5.0 we can enforce a min/max number of positional args
+        List<String> args = arguments.values(options);
+        if (args.size() != 1) {
+            throw new UserError(ExitCodes.USAGE, "Must supply a single plugin id argument");
+        }
+        execute(terminal, args.get(0));
+    }
+
+    // pkg private for testing
+    void execute(Terminal terminal, String pluginName) throws Exception {
         terminal.println("-> Removing " + Strings.coalesceToEmpty(pluginName) + "...");
 
         Path pluginDir = env.pluginsFile().resolve(pluginName);
         if (Files.exists(pluginDir) == false) {
-            throw new UserError(CliTool.ExitStatus.USAGE, "Plugin " + pluginName + " not found. Run 'plugin list' to get list of installed plugins.");
+            throw new UserError(ExitCodes.USAGE, "Plugin " + pluginName + " not found. Run 'plugin list' to get list of installed plugins.");
         }
 
         List<Path> pluginPaths = new ArrayList<>();
@@ -60,7 +75,7 @@ class RemovePluginCommand extends CliTool.Command {
         Path pluginBinDir = env.binFile().resolve(pluginName);
         if (Files.exists(pluginBinDir)) {
             if (Files.isDirectory(pluginBinDir) == false) {
-                throw new UserError(CliTool.ExitStatus.IO_ERROR, "Bin dir for " + pluginName + " is not a directory");
+                throw new UserError(ExitCodes.IO_ERROR, "Bin dir for " + pluginName + " is not a directory");
             }
             pluginPaths.add(pluginBinDir);
             terminal.println(VERBOSE, "Removing: " + pluginBinDir);
@@ -72,7 +87,5 @@ class RemovePluginCommand extends CliTool.Command {
         pluginPaths.add(tmpPluginDir);
 
         IOUtils.rm(pluginPaths.toArray(new Path[pluginPaths.size()]));
-
-        return CliTool.ExitStatus.OK;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
index 6eb32cfb06f..da2d9688095 100644
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
+++ b/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
@@ -22,7 +22,6 @@ package org.elasticsearch.repositories;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest;
@@ -31,6 +30,7 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.inject.Injector;
diff --git a/core/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java b/core/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
index 91600488332..48ffbd5c1cb 100644
--- a/core/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
@@ -22,9 +22,9 @@ package org.elasticsearch.repositories;
 import com.carrotsearch.hppc.ObjectContainer;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index a6ea381adb4..5d423552a56 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -478,7 +478,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
                 metaDataBuilder.put(indexMetaDataFormat(snapshotVersion).read(indexMetaDataBlobContainer, snapshotId.getSnapshot()), false);
             } catch (ElasticsearchParseException | IOException ex) {
                 if (ignoreIndexErrors) {
-                    logger.warn("[{}] [{}] failed to read metadata for index", snapshotId, index, ex);
+                    logger.warn("[{}] [{}] failed to read metadata for index", ex, snapshotId, index);
                 } else {
                     throw ex;
                 }
diff --git a/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java b/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
index 0aa62225479..56e7e08c2c3 100644
--- a/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.blobstore.fs.FsBlobStore;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.snapshots.IndexShardRepository;
@@ -51,12 +52,17 @@ public class FsRepository extends BlobStoreRepository {
 
     public final static String TYPE = "fs";
 
-    public static final Setting<String> LOCATION_SETTING = new Setting<>("location", "", Function.identity(), false, Setting.Scope.CLUSTER);
-    public static final Setting<String> REPOSITORIES_LOCATION_SETTING = new Setting<>("repositories.fs.location", LOCATION_SETTING, Function.identity(), false, Setting.Scope.CLUSTER);
-    public static final Setting<ByteSizeValue> CHUNK_SIZE_SETTING = Setting.byteSizeSetting("chunk_size", "-1", false, Setting.Scope.CLUSTER);
-    public static final Setting<ByteSizeValue> REPOSITORIES_CHUNK_SIZE_SETTING = Setting.byteSizeSetting("repositories.fs.chunk_size", "-1", false, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("compress", false, false, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> REPOSITORIES_COMPRESS_SETTING = Setting.boolSetting("repositories.fs.compress", false, false, Setting.Scope.CLUSTER);
+    public static final Setting<String> LOCATION_SETTING =
+        new Setting<>("location", "", Function.identity(), Property.NodeScope);
+    public static final Setting<String> REPOSITORIES_LOCATION_SETTING =
+        new Setting<>("repositories.fs.location", LOCATION_SETTING, Function.identity(), Property.NodeScope);
+    public static final Setting<ByteSizeValue> CHUNK_SIZE_SETTING =
+        Setting.byteSizeSetting("chunk_size", "-1", Property.NodeScope);
+    public static final Setting<ByteSizeValue> REPOSITORIES_CHUNK_SIZE_SETTING =
+        Setting.byteSizeSetting("repositories.fs.chunk_size", "-1", Property.NodeScope);
+    public static final Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("compress", false, Property.NodeScope);
+    public static final Setting<Boolean> REPOSITORIES_COMPRESS_SETTING =
+        Setting.boolSetting("repositories.fs.compress", false, Property.NodeScope);
 
     private final FsBlobStore blobStore;
 
diff --git a/core/src/main/java/org/elasticsearch/repositories/uri/URLIndexShardRepository.java b/core/src/main/java/org/elasticsearch/repositories/uri/URLIndexShardRepository.java
index ab9ec72463a..616a36d5066 100644
--- a/core/src/main/java/org/elasticsearch/repositories/uri/URLIndexShardRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/uri/URLIndexShardRepository.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.repositories.uri;
 
-import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
diff --git a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
index 2d15db245aa..77d4f1cc816 100644
--- a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.blobstore.url.URLBlobStore;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.util.URIPattern;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.snapshots.IndexShardRepository;
@@ -55,19 +56,22 @@ public class URLRepository extends BlobStoreRepository {
 
     public final static String TYPE = "url";
 
-    public static final Setting<List<String>> SUPPORTED_PROTOCOLS_SETTING = Setting.listSetting("repositories.url.supported_protocols",
-            Arrays.asList("http", "https", "ftp", "file", "jar"), Function.identity(), false, Setting.Scope.CLUSTER);
+    public static final Setting<List<String>> SUPPORTED_PROTOCOLS_SETTING =
+        Setting.listSetting("repositories.url.supported_protocols", Arrays.asList("http", "https", "ftp", "file", "jar"),
+            Function.identity(), Property.NodeScope);
 
-    public static final Setting<List<URIPattern>> ALLOWED_URLS_SETTING = Setting.listSetting("repositories.url.allowed_urls",
-            Collections.emptyList(), URIPattern::new, false, Setting.Scope.CLUSTER);
+    public static final Setting<List<URIPattern>> ALLOWED_URLS_SETTING =
+        Setting.listSetting("repositories.url.allowed_urls", Collections.emptyList(), URIPattern::new, Property.NodeScope);
 
-    public static final Setting<URL> URL_SETTING = new Setting<>("url", "http:", URLRepository::parseURL, false, Setting.Scope.CLUSTER);
-    public static final Setting<URL> REPOSITORIES_URL_SETTING = new Setting<>("repositories.url.url", (s) -> s.get("repositories.uri.url", "http:"),
-            URLRepository::parseURL, false, Setting.Scope.CLUSTER);
+    public static final Setting<URL> URL_SETTING = new Setting<>("url", "http:", URLRepository::parseURL, Property.NodeScope);
+    public static final Setting<URL> REPOSITORIES_URL_SETTING =
+        new Setting<>("repositories.url.url", (s) -> s.get("repositories.uri.url", "http:"), URLRepository::parseURL,
+            Property.NodeScope);
 
-    public static final Setting<Boolean> LIST_DIRECTORIES_SETTING = Setting.boolSetting("list_directories", true, false, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> REPOSITORIES_LIST_DIRECTORIES_SETTING = Setting.boolSetting("repositories.uri.list_directories", true,
-            false, Setting.Scope.CLUSTER);
+    public static final Setting<Boolean> LIST_DIRECTORIES_SETTING =
+        Setting.boolSetting("list_directories", true, Property.NodeScope);
+    public static final Setting<Boolean> REPOSITORIES_LIST_DIRECTORIES_SETTING =
+        Setting.boolSetting("repositories.uri.list_directories", true, Property.NodeScope);
 
     private final List<String> supportedProtocols;
 
diff --git a/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
index 60b3ccce930..b406dfca545 100644
--- a/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
+++ b/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
@@ -23,6 +23,7 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 /**
@@ -34,7 +35,8 @@ import org.elasticsearch.common.settings.Settings;
  * {@link org.elasticsearch.rest.RestController#registerRelevantHeaders(String...)}
  */
 public abstract class BaseRestHandler extends AbstractComponent implements RestHandler {
-    public static final Setting<Boolean> MULTI_ALLOW_EXPLICIT_INDEX = Setting.boolSetting("rest.action.multi.allow_explicit_index", true, false, Setting.Scope.CLUSTER);
+    public static final Setting<Boolean> MULTI_ALLOW_EXPLICIT_INDEX =
+        Setting.boolSetting("rest.action.multi.allow_explicit_index", true, Property.NodeScope);
     private final Client client;
     protected final ParseFieldMatcher parseFieldMatcher;
 
diff --git a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
index ac8eadade0b..52f624849fc 100644
--- a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
+++ b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
@@ -126,7 +126,11 @@ public class BytesRestResponse extends RestResponse {
             if (channel.request().paramAsBoolean("error_trace", !ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE_DEFAULT)) {
                 params =  new ToXContent.DelegatingMapParams(Collections.singletonMap(ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE, "false"), channel.request());
             } else {
-                SUPPRESSED_ERROR_LOGGER.info("{} Params: {}", t, channel.request().path(), channel.request().params());
+                if (status.getStatus() < 500) {
+                    SUPPRESSED_ERROR_LOGGER.debug("{} Params: {}", t, channel.request().path(), channel.request().params());
+                } else {
+                    SUPPRESSED_ERROR_LOGGER.warn("{} Params: {}", t, channel.request().path(), channel.request().params());
+                }
                 params = channel.request();
             }
             builder.field("error");
diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java
index 64e21002d8c..0cbfdd0ef1b 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestController.java
@@ -176,7 +176,7 @@ public class RestController extends AbstractLifecycleComponent<RestController> {
                     try {
                         channel.sendResponse(new BytesRestResponse(channel, e));
                     } catch (Throwable e1) {
-                        logger.error("failed to send failure response for uri [" + request.uri() + "]", e1);
+                        logger.error("failed to send failure response for uri [{}]", e1, request.uri());
                     }
                 }
             } else {
@@ -275,7 +275,7 @@ public class RestController extends AbstractLifecycleComponent<RestController> {
                 try {
                     channel.sendResponse(new BytesRestResponse(channel, e));
                 } catch (IOException e1) {
-                    logger.error("Failed to send failure response for uri [" + request.uri() + "]", e1);
+                    logger.error("Failed to send failure response for uri [{}]", e1, request.uri());
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java
index fb8e9c63740..1e2aece1646 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java
@@ -81,6 +81,7 @@ public class RestNodesStatsAction extends BaseRestHandler {
             nodesStatsRequest.breaker(metrics.contains("breaker"));
             nodesStatsRequest.script(metrics.contains("script"));
             nodesStatsRequest.discovery(metrics.contains("discovery"));
+            nodesStatsRequest.ingest(metrics.contains("ingest"));
 
             // check for index specific metrics
             if (metrics.contains("indices")) {
@@ -113,6 +114,6 @@ public class RestNodesStatsAction extends BaseRestHandler {
             nodesStatsRequest.indices().includeSegmentFileSizes(true);
         }
 
-        client.admin().cluster().nodesStats(nodesStatsRequest, new RestToXContentListener<NodesStatsResponse>(channel));
+        client.admin().cluster().nodesStats(nodesStatsRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java
index 99cdc16253a..658090bb6db 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java
@@ -52,10 +52,10 @@ public class RestCancelTasksAction extends BaseRestHandler {
         TaskId parentTaskId = new TaskId(request.param("parent_task_id"));
 
         CancelTasksRequest cancelTasksRequest = new CancelTasksRequest();
-        cancelTasksRequest.taskId(taskId);
-        cancelTasksRequest.nodesIds(nodesIds);
-        cancelTasksRequest.actions(actions);
-        cancelTasksRequest.parentTaskId(parentTaskId);
+        cancelTasksRequest.setTaskId(taskId);
+        cancelTasksRequest.setNodesIds(nodesIds);
+        cancelTasksRequest.setActions(actions);
+        cancelTasksRequest.setParentTaskId(parentTaskId);
         client.admin().cluster().cancelTasks(cancelTasksRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java
index 992267fa8a5..168d091968e 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java
@@ -24,6 +24,7 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
@@ -50,13 +51,17 @@ public class RestListTasksAction extends BaseRestHandler {
         TaskId taskId = new TaskId(request.param("taskId"));
         String[] actions = Strings.splitStringByCommaToArray(request.param("actions"));
         TaskId parentTaskId = new TaskId(request.param("parent_task_id"));
+        boolean waitForCompletion = request.paramAsBoolean("wait_for_completion", false);
+        TimeValue timeout = request.paramAsTime("timeout", null);
 
         ListTasksRequest listTasksRequest = new ListTasksRequest();
-        listTasksRequest.taskId(taskId);
-        listTasksRequest.nodesIds(nodesIds);
-        listTasksRequest.detailed(detailed);
-        listTasksRequest.actions(actions);
-        listTasksRequest.parentTaskId(parentTaskId);
+        listTasksRequest.setTaskId(taskId);
+        listTasksRequest.setNodesIds(nodesIds);
+        listTasksRequest.setDetailed(detailed);
+        listTasksRequest.setActions(actions);
+        listTasksRequest.setParentTaskId(parentTaskId);
+        listTasksRequest.setWaitForCompletion(waitForCompletion);
+        listTasksRequest.setTimeout(timeout);
         client.admin().cluster().listTasks(listTasksRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestUpdateSettingsAction.java
index bcf43a4baa6..4477ff25011 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestUpdateSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestUpdateSettingsAction.java
@@ -47,6 +47,7 @@ public class RestUpdateSettingsAction extends BaseRestHandler {
             "timeout",
             "master_timeout",
             "index",
+            "preserve_existing",
             "expand_wildcards",
             "ignore_unavailable",
             "allow_no_indices"));
@@ -62,6 +63,7 @@ public class RestUpdateSettingsAction extends BaseRestHandler {
     public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
         UpdateSettingsRequest updateSettingsRequest = updateSettingsRequest(Strings.splitStringByCommaToArray(request.param("index")));
         updateSettingsRequest.timeout(request.paramAsTime("timeout", updateSettingsRequest.timeout()));
+        updateSettingsRequest.setPreserveExisting(request.paramAsBoolean("preserve_existing", updateSettingsRequest.isPreserveExisting()));
         updateSettingsRequest.masterNodeTimeout(request.paramAsTime("master_timeout", updateSettingsRequest.masterNodeTimeout()));
         updateSettingsRequest.indicesOptions(IndicesOptions.fromRequest(request, updateSettingsRequest.indicesOptions()));
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java
index fa4371846f6..dbda83709ba 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java
@@ -71,18 +71,17 @@ public class RestIndicesStatsAction extends BaseRestHandler {
             indicesStatsRequest.docs(metrics.contains("docs"));
             indicesStatsRequest.store(metrics.contains("store"));
             indicesStatsRequest.indexing(metrics.contains("indexing"));
-            indicesStatsRequest.search(metrics.contains("search"));
+            indicesStatsRequest.search(metrics.contains("search") || metrics.contains("suggest"));
             indicesStatsRequest.get(metrics.contains("get"));
             indicesStatsRequest.merge(metrics.contains("merge"));
             indicesStatsRequest.refresh(metrics.contains("refresh"));
             indicesStatsRequest.flush(metrics.contains("flush"));
             indicesStatsRequest.warmer(metrics.contains("warmer"));
             indicesStatsRequest.queryCache(metrics.contains("query_cache"));
-            indicesStatsRequest.percolate(metrics.contains("percolate"));
+            indicesStatsRequest.percolate(metrics.contains("percolator_cache"));
             indicesStatsRequest.segments(metrics.contains("segments"));
             indicesStatsRequest.fieldData(metrics.contains("fielddata"));
             indicesStatsRequest.completion(metrics.contains("completion"));
-            indicesStatsRequest.suggest(metrics.contains("suggest"));
             indicesStatsRequest.requestCache(metrics.contains("request_cache"));
             indicesStatsRequest.recovery(metrics.contains("recovery"));
             indicesStatsRequest.translog(metrics.contains("translog"));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index aec087523b8..958fa40b54b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -83,8 +83,8 @@ public class RestIndicesAction extends AbstractCatAction {
             public void processResponse(final ClusterStateResponse clusterStateResponse) {
                 ClusterState state = clusterStateResponse.getState();
                 final IndicesOptions concreteIndicesOptions = IndicesOptions.fromOptions(false, true, true, true);
-                final String[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, concreteIndicesOptions, indices);
-                final String[] openIndices = indexNameExpressionResolver.concreteIndices(state, IndicesOptions.lenientExpandOpen(), indices);
+                final String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, concreteIndicesOptions, indices);
+                final String[] openIndices = indexNameExpressionResolver.concreteIndexNames(state, IndicesOptions.lenientExpandOpen(), indices);
                 ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest(openIndices);
                 clusterHealthRequest.local(request.paramAsBoolean("local", clusterHealthRequest.local()));
                 client.admin().cluster().health(clusterHealthRequest, new RestActionListener<ClusterHealthResponse>(channel) {
@@ -135,22 +135,22 @@ public class RestIndicesAction extends AbstractCatAction {
         table.addCell("fielddata.evictions", "sibling:pri;alias:fe,fielddataEvictions;default:false;text-align:right;desc:fielddata evictions");
         table.addCell("pri.fielddata.evictions", "default:false;text-align:right;desc:fielddata evictions");
 
-        table.addCell("query_cache.memory_size", "sibling:pri;alias:fcm,queryCacheMemory;default:false;text-align:right;desc:used query cache");
+        table.addCell("query_cache.memory_size", "sibling:pri;alias:qcm,queryCacheMemory;default:false;text-align:right;desc:used query cache");
         table.addCell("pri.query_cache.memory_size", "default:false;text-align:right;desc:used query cache");
 
-        table.addCell("query_cache.evictions", "sibling:pri;alias:fce,queryCacheEvictions;default:false;text-align:right;desc:query cache evictions");
+        table.addCell("query_cache.evictions", "sibling:pri;alias:qce,queryCacheEvictions;default:false;text-align:right;desc:query cache evictions");
         table.addCell("pri.query_cache.evictions", "default:false;text-align:right;desc:query cache evictions");
 
-        table.addCell("request_cache.memory_size", "sibling:pri;alias:qcm,queryCacheMemory;default:false;text-align:right;desc:used request cache");
+        table.addCell("request_cache.memory_size", "sibling:pri;alias:rcm,requestCacheMemory;default:false;text-align:right;desc:used request cache");
         table.addCell("pri.request_cache.memory_size", "default:false;text-align:right;desc:used request cache");
 
-        table.addCell("request_cache.evictions", "sibling:pri;alias:qce,queryCacheEvictions;default:false;text-align:right;desc:request cache evictions");
+        table.addCell("request_cache.evictions", "sibling:pri;alias:rce,requestCacheEvictions;default:false;text-align:right;desc:request cache evictions");
         table.addCell("pri.request_cache.evictions", "default:false;text-align:right;desc:request cache evictions");
 
-        table.addCell("request_cache.hit_count", "sibling:pri;alias:qchc,queryCacheHitCount;default:false;text-align:right;desc:request cache hit count");
+        table.addCell("request_cache.hit_count", "sibling:pri;alias:rchc,requestCacheHitCount;default:false;text-align:right;desc:request cache hit count");
         table.addCell("pri.request_cache.hit_count", "default:false;text-align:right;desc:request cache hit count");
 
-        table.addCell("request_cache.miss_count", "sibling:pri;alias:qcmc,queryCacheMissCount;default:false;text-align:right;desc:request cache miss count");
+        table.addCell("request_cache.miss_count", "sibling:pri;alias:rcmc,requestCacheMissCount;default:false;text-align:right;desc:request cache miss count");
         table.addCell("pri.request_cache.miss_count", "default:false;text-align:right;desc:request cache miss count");
 
         table.addCell("flush.total", "sibling:pri;alias:ft,flushTotal;default:false;text-align:right;desc:number of flushes");
@@ -222,21 +222,9 @@ public class RestIndicesAction extends AbstractCatAction {
         table.addCell("merges.total_time", "sibling:pri;alias:mtt,mergesTotalTime;default:false;text-align:right;desc:time spent in merges");
         table.addCell("pri.merges.total_time", "default:false;text-align:right;desc:time spent in merges");
 
-        table.addCell("percolate.current", "sibling:pri;alias:pc,percolateCurrent;default:false;text-align:right;desc:number of current percolations");
-        table.addCell("pri.percolate.current", "default:false;text-align:right;desc:number of current percolations");
-
-        table.addCell("percolate.memory_size", "sibling:pri;alias:pm,percolateMemory;default:false;text-align:right;desc:memory used by percolations");
-        table.addCell("pri.percolate.memory_size", "default:false;text-align:right;desc:memory used by percolations");
-
         table.addCell("percolate.queries", "sibling:pri;alias:pq,percolateQueries;default:false;text-align:right;desc:number of registered percolation queries");
         table.addCell("pri.percolate.queries", "default:false;text-align:right;desc:number of registered percolation queries");
 
-        table.addCell("percolate.time", "sibling:pri;alias:pti,percolateTime;default:false;text-align:right;desc:time spent percolating");
-        table.addCell("pri.percolate.time", "default:false;text-align:right;desc:time spent percolating");
-
-        table.addCell("percolate.total", "sibling:pri;alias:pto,percolateTotal;default:false;text-align:right;desc:total percolations");
-        table.addCell("pri.percolate.total", "default:false;text-align:right;desc:total percolations");
-
         table.addCell("refresh.total", "sibling:pri;alias:rto,refreshTotal;default:false;text-align:right;desc:total refreshes");
         table.addCell("pri.refresh.total", "default:false;text-align:right;desc:total refreshes");
 
@@ -436,20 +424,8 @@ public class RestIndicesAction extends AbstractCatAction {
             table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getTotalTime());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getTotalTime());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getPercolate().getCurrent());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getPercolate().getCurrent());
-
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getPercolate().getMemorySize());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getPercolate().getMemorySize());
-
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getPercolate().getNumQueries());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getPercolate().getNumQueries());
-
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getPercolate().getTime());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getPercolate().getTime());
-
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getPercolate().getCount());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getPercolate().getCount());
+            table.addCell(indexStats == null ? null : indexStats.getTotal().getPercolatorCache().getNumQueries());
+            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getPercolatorCache().getNumQueries());
 
             table.addCell(indexStats == null ? null : indexStats.getTotal().getRefresh().getTotal());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRefresh().getTotal());
@@ -514,14 +490,14 @@ public class RestIndicesAction extends AbstractCatAction {
             table.addCell(indexStats == null ? null : indexStats.getTotal().getWarmer().totalTime());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getWarmer().totalTime());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSuggest().getCurrent());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSuggest().getCurrent());
+            table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getSuggestCurrent());
+            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getSuggestCurrent());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSuggest().getTime());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSuggest().getTime());
+            table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getSuggestTime());
+            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getSuggestTime());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSuggest().getCount());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSuggest().getCount());
+            table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getSuggestCount());
+            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getSuggestCount());
 
             table.addCell(indexStats == null ? null : indexStats.getTotal().getTotalMemory());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getTotalMemory());
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
index 0605bc4dcab..54ee855b543 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
@@ -46,10 +46,9 @@ import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.get.GetStats;
 import org.elasticsearch.index.shard.IndexingStats;
 import org.elasticsearch.index.merge.MergeStats;
-import org.elasticsearch.index.percolator.PercolateStats;
+import org.elasticsearch.index.percolator.PercolatorQueryCacheStats;
 import org.elasticsearch.index.refresh.RefreshStats;
 import org.elasticsearch.index.search.stats.SearchStats;
-import org.elasticsearch.index.suggest.stats.SuggestStats;
 import org.elasticsearch.indices.NodeIndicesStats;
 import org.elasticsearch.monitor.fs.FsInfo;
 import org.elasticsearch.monitor.jvm.JvmInfo;
@@ -67,7 +66,6 @@ import org.elasticsearch.script.ScriptStats;
 import org.elasticsearch.search.suggest.completion.CompletionStats;
 
 import java.util.Locale;
-import java.util.Map;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
@@ -151,13 +149,13 @@ public class RestNodesAction extends AbstractCatAction {
         table.addCell("fielddata.memory_size", "alias:fm,fielddataMemory;default:false;text-align:right;desc:used fielddata cache");
         table.addCell("fielddata.evictions", "alias:fe,fielddataEvictions;default:false;text-align:right;desc:fielddata evictions");
 
-        table.addCell("query_cache.memory_size", "alias:fcm,queryCacheMemory;default:false;text-align:right;desc:used query cache");
-        table.addCell("query_cache.evictions", "alias:fce,queryCacheEvictions;default:false;text-align:right;desc:query cache evictions");
+        table.addCell("query_cache.memory_size", "alias:qcm,queryCacheMemory;default:false;text-align:right;desc:used query cache");
+        table.addCell("query_cache.evictions", "alias:qce,queryCacheEvictions;default:false;text-align:right;desc:query cache evictions");
 
-        table.addCell("request_cache.memory_size", "alias:qcm,requestCacheMemory;default:false;text-align:right;desc:used request cache");
-        table.addCell("request_cache.evictions", "alias:qce,requestCacheEvictions;default:false;text-align:right;desc:request cache evictions");
-        table.addCell("request_cache.hit_count", "alias:qchc,requestCacheHitCount;default:false;text-align:right;desc:request cache hit counts");
-        table.addCell("request_cache.miss_count", "alias:qcmc,requestCacheMissCount;default:false;text-align:right;desc:request cache miss counts");
+        table.addCell("request_cache.memory_size", "alias:rcm,requestCacheMemory;default:false;text-align:right;desc:used request cache");
+        table.addCell("request_cache.evictions", "alias:rce,requestCacheEvictions;default:false;text-align:right;desc:request cache evictions");
+        table.addCell("request_cache.hit_count", "alias:rchc,requestCacheHitCount;default:false;text-align:right;desc:request cache hit counts");
+        table.addCell("request_cache.miss_count", "alias:rcmc,requestCacheMissCount;default:false;text-align:right;desc:request cache miss counts");
 
         table.addCell("flush.total", "alias:ft,flushTotal;default:false;text-align:right;desc:number of flushes");
         table.addCell("flush.total_time", "alias:ftt,flushTotalTime;default:false;text-align:right;desc:time spent in flush");
@@ -186,11 +184,7 @@ public class RestNodesAction extends AbstractCatAction {
         table.addCell("merges.total_size", "alias:mts,mergesTotalSize;default:false;text-align:right;desc:size merged");
         table.addCell("merges.total_time", "alias:mtt,mergesTotalTime;default:false;text-align:right;desc:time spent in merges");
 
-        table.addCell("percolate.current", "alias:pc,percolateCurrent;default:false;text-align:right;desc:number of current percolations");
-        table.addCell("percolate.memory_size", "alias:pm,percolateMemory;default:false;text-align:right;desc:memory used by percolations");
         table.addCell("percolate.queries", "alias:pq,percolateQueries;default:false;text-align:right;desc:number of registered percolation queries");
-        table.addCell("percolate.time", "alias:pti,percolateTime;default:false;text-align:right;desc:time spent percolating");
-        table.addCell("percolate.total", "alias:pto,percolateTotal;default:false;text-align:right;desc:total percolations");
 
         table.addCell("refresh.total", "alias:rto,refreshTotal;default:false;text-align:right;desc:total refreshes");
         table.addCell("refresh.time", "alias:rti,refreshTime;default:false;text-align:right;desc:time spent in refreshes");
@@ -336,12 +330,8 @@ public class RestNodesAction extends AbstractCatAction {
             table.addCell(mergeStats == null ? null : mergeStats.getTotalSize());
             table.addCell(mergeStats == null ? null : mergeStats.getTotalTime());
 
-            PercolateStats percolateStats = indicesStats == null ? null : indicesStats.getPercolate();
-            table.addCell(percolateStats == null ? null : percolateStats.getCurrent());
-            table.addCell(percolateStats == null ? null : percolateStats.getMemorySize());
-            table.addCell(percolateStats == null ? null : percolateStats.getNumQueries());
-            table.addCell(percolateStats == null ? null : percolateStats.getTime());
-            table.addCell(percolateStats == null ? null : percolateStats.getCount());
+            PercolatorQueryCacheStats percolatorQueryCacheStats = indicesStats == null ? null : indicesStats.getPercolate();
+            table.addCell(percolatorQueryCacheStats == null ? null : percolatorQueryCacheStats.getNumQueries());
 
             RefreshStats refreshStats = indicesStats == null ? null : indicesStats.getRefresh();
             table.addCell(refreshStats == null ? null : refreshStats.getTotal());
@@ -371,10 +361,9 @@ public class RestNodesAction extends AbstractCatAction {
             table.addCell(segmentsStats == null ? null : segmentsStats.getVersionMapMemory());
             table.addCell(segmentsStats == null ? null : segmentsStats.getBitsetMemory());
 
-            SuggestStats suggestStats = indicesStats == null ? null : indicesStats.getSuggest();
-            table.addCell(suggestStats == null ? null : suggestStats.getCurrent());
-            table.addCell(suggestStats == null ? null : suggestStats.getTime());
-            table.addCell(suggestStats == null ? null : suggestStats.getCount());
+            table.addCell(searchStats == null ? null : searchStats.getTotal().getSuggestCurrent());
+            table.addCell(searchStats == null ? null : searchStats.getTotal().getSuggestTime());
+            table.addCell(searchStats == null ? null : searchStats.getTotal().getSuggestCount());
 
             table.endRow();
         }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
index 759fac2eb19..7c555c9b357 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
@@ -92,14 +92,16 @@ public class RestRecoveryAction extends AbstractCatAction {
                 .addCell("repository", "alias:rep;desc:repository")
                 .addCell("snapshot", "alias:snap;desc:snapshot")
                 .addCell("files", "alias:f;desc:number of files to recover")
+                .addCell("files_recovered", "alias:fr;desc:files recovered")
                 .addCell("files_percent", "alias:fp;desc:percent of files recovered")
-                .addCell("bytes", "alias:b;desc:size to recover in bytes")
+                .addCell("files_total", "alias:tf;desc:total number of files")
+                .addCell("bytes", "alias:b;desc:number of bytes to recover")
+                .addCell("bytes_recovered", "alias:br;desc:bytes recovered")
                 .addCell("bytes_percent", "alias:bp;desc:percent of bytes recovered")
-                .addCell("total_files", "alias:tf;desc:total number of files")
-                .addCell("total_bytes", "alias:tb;desc:total number of bytes")
-                .addCell("translog", "alias:tr;desc:translog operations recovered")
-                .addCell("translog_percent", "alias:trp;desc:percent of translog recovery")
-                .addCell("total_translog", "alias:trt;desc:current total translog operations")
+                .addCell("bytes_total", "alias:tb;desc:total number of bytes")
+                .addCell("translog_ops", "alias:to;desc:number of translog ops to recover")
+                .addCell("translog_ops_recovered", "alias:tor;desc:translog ops recovered")
+                .addCell("translog_ops_percent", "alias:top;desc:percent of translog ops recovered")
                 .endHeaders();
         return t;
     }
@@ -151,14 +153,16 @@ public class RestRecoveryAction extends AbstractCatAction {
                 t.addCell(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshotId().getRepository());
                 t.addCell(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshotId().getSnapshot());
                 t.addCell(state.getIndex().totalRecoverFiles());
+                t.addCell(state.getIndex().recoveredFileCount());
                 t.addCell(String.format(Locale.ROOT, "%1.1f%%", state.getIndex().recoveredFilesPercent()));
-                t.addCell(state.getIndex().totalRecoverBytes());
-                t.addCell(String.format(Locale.ROOT, "%1.1f%%", state.getIndex().recoveredBytesPercent()));
                 t.addCell(state.getIndex().totalFileCount());
+                t.addCell(state.getIndex().totalRecoverBytes());
+                t.addCell(state.getIndex().recoveredBytes());
+                t.addCell(String.format(Locale.ROOT, "%1.1f%%", state.getIndex().recoveredBytesPercent()));
                 t.addCell(state.getIndex().totalBytes());
+                t.addCell(state.getTranslog().totalOperations());
                 t.addCell(state.getTranslog().recoveredOperations());
                 t.addCell(String.format(Locale.ROOT, "%1.1f%%", state.getTranslog().recoveredPercent()));
-                t.addCell(state.getTranslog().totalOperations());
                 t.endRow();
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
index 94a82e8e773..8bf67653f6f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
@@ -109,8 +109,8 @@ public class RestShardsAction extends AbstractCatAction {
         table.addCell("fielddata.memory_size", "alias:fm,fielddataMemory;default:false;text-align:right;desc:used fielddata cache");
         table.addCell("fielddata.evictions", "alias:fe,fielddataEvictions;default:false;text-align:right;desc:fielddata evictions");
 
-        table.addCell("query_cache.memory_size", "alias:fcm,queryCacheMemory;default:false;text-align:right;desc:used query cache");
-        table.addCell("query_cache.evictions", "alias:fce,queryCacheEvictions;default:false;text-align:right;desc:query cache evictions");
+        table.addCell("query_cache.memory_size", "alias:qcm,queryCacheMemory;default:false;text-align:right;desc:used query cache");
+        table.addCell("query_cache.evictions", "alias:qce,queryCacheEvictions;default:false;text-align:right;desc:query cache evictions");
 
         table.addCell("flush.total", "alias:ft,flushTotal;default:false;text-align:right;desc:number of flushes");
         table.addCell("flush.total_time", "alias:ftt,flushTotalTime;default:false;text-align:right;desc:time spent in flush");
@@ -139,11 +139,7 @@ public class RestShardsAction extends AbstractCatAction {
         table.addCell("merges.total_size", "alias:mts,mergesTotalSize;default:false;text-align:right;desc:size merged");
         table.addCell("merges.total_time", "alias:mtt,mergesTotalTime;default:false;text-align:right;desc:time spent in merges");
 
-        table.addCell("percolate.current", "alias:pc,percolateCurrent;default:false;text-align:right;desc:number of current percolations");
-        table.addCell("percolate.memory_size", "alias:pm,percolateMemory;default:false;text-align:right;desc:memory used by percolations");
         table.addCell("percolate.queries", "alias:pq,percolateQueries;default:false;text-align:right;desc:number of registered percolation queries");
-        table.addCell("percolate.time", "alias:pti,percolateTime;default:false;text-align:right;desc:time spent percolating");
-        table.addCell("percolate.total", "alias:pto,percolateTotal;default:false;text-align:right;desc:total percolations");
 
         table.addCell("refresh.total", "alias:rto,refreshTotal;default:false;text-align:right;desc:total refreshes");
         table.addCell("refresh.time", "alias:rti,refreshTime;default:false;text-align:right;desc:time spent in refreshes");
@@ -191,7 +187,7 @@ public class RestShardsAction extends AbstractCatAction {
             table.addCell(shard.getIndexName());
             table.addCell(shard.id());
 
-            IndexMetaData indexMeta = state.getState().getMetaData().index(shard.index());
+            IndexMetaData indexMeta = state.getState().getMetaData().getIndexSafe(shard.index());
             boolean usesShadowReplicas = false;
             if (indexMeta != null) {
                 usesShadowReplicas = IndexMetaData.isIndexUsingShadowReplicas(indexMeta.getSettings());
@@ -282,11 +278,7 @@ public class RestShardsAction extends AbstractCatAction {
             table.addCell(commonStats == null ? null : commonStats.getMerge().getTotalSize());
             table.addCell(commonStats == null ? null : commonStats.getMerge().getTotalTime());
 
-            table.addCell(commonStats == null ? null : commonStats.getPercolate().getCurrent());
-            table.addCell(commonStats == null ? null : commonStats.getPercolate().getMemorySize());
-            table.addCell(commonStats == null ? null : commonStats.getPercolate().getNumQueries());
-            table.addCell(commonStats == null ? null : commonStats.getPercolate().getTime());
-            table.addCell(commonStats == null ? null : commonStats.getPercolate().getCount());
+            table.addCell(commonStats == null ? null : commonStats.getPercolatorCache().getNumQueries());
 
             table.addCell(commonStats == null ? null : commonStats.getRefresh().getTotal());
             table.addCell(commonStats == null ? null : commonStats.getRefresh().getTotalTime());
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
index fa2e662c738..b4ba4bbeffa 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
@@ -63,11 +63,9 @@ public class RestThreadPoolAction extends AbstractCatAction {
             ThreadPool.Names.INDEX,
             ThreadPool.Names.MANAGEMENT,
             ThreadPool.Names.FORCE_MERGE,
-            ThreadPool.Names.PERCOLATE,
             ThreadPool.Names.REFRESH,
             ThreadPool.Names.SEARCH,
             ThreadPool.Names.SNAPSHOT,
-            ThreadPool.Names.SUGGEST,
             ThreadPool.Names.WARMER
     };
 
@@ -79,11 +77,9 @@ public class RestThreadPoolAction extends AbstractCatAction {
             "i",
             "ma",
             "fm",
-            "p",
             "r",
             "s",
             "sn",
-            "su",
             "w"
     };
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java b/core/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java
index bf3f0a3e5df..205bea92f96 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java
@@ -23,7 +23,7 @@ import org.elasticsearch.Build;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
index 5d9ac118831..10258aaaee4 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
@@ -44,6 +44,7 @@ import org.elasticsearch.rest.action.support.RestToXContentListener;
 import org.elasticsearch.script.Template;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.suggest.Suggesters;
 
 import java.util.Map;
 
@@ -60,13 +61,14 @@ public class RestMultiSearchAction extends BaseRestHandler {
     private final boolean allowExplicitIndex;
     private final IndicesQueriesRegistry indicesQueriesRegistry;
     private final AggregatorParsers aggParsers;
-
+    private final Suggesters suggesters;
 
     @Inject
     public RestMultiSearchAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry indicesQueriesRegistry,
-            AggregatorParsers aggParsers) {
+                                 AggregatorParsers aggParsers, Suggesters suggesters) {
         super(settings, client);
         this.aggParsers = aggParsers;
+        this.suggesters = suggesters;
 
         controller.registerHandler(GET, "/_msearch", this);
         controller.registerHandler(POST, "/_msearch", this);
@@ -97,7 +99,7 @@ public class RestMultiSearchAction extends BaseRestHandler {
         IndicesOptions indicesOptions = IndicesOptions.fromRequest(request, multiSearchRequest.indicesOptions());
         parseRequest(multiSearchRequest, RestActions.getRestContent(request), isTemplateRequest, indices, types,
                 request.param("search_type"), request.param("routing"), indicesOptions, allowExplicitIndex, indicesQueriesRegistry,
-                parseFieldMatcher, aggParsers);
+                parseFieldMatcher, aggParsers, suggesters);
         client.multiSearch(multiSearchRequest, new RestToXContentListener<>(channel));
     }
 
@@ -112,7 +114,8 @@ public class RestMultiSearchAction extends BaseRestHandler {
                                                    @Nullable String routing,
                                                    IndicesOptions indicesOptions,
                                                    boolean allowExplicitIndex, IndicesQueriesRegistry indicesQueriesRegistry,
-            ParseFieldMatcher parseFieldMatcher, AggregatorParsers aggParsers) throws Exception {
+                                                   ParseFieldMatcher parseFieldMatcher, AggregatorParsers aggParsers,
+                                                   Suggesters suggesters) throws Exception {
         XContent xContent = XContentFactory.xContent(data);
         int from = 0;
         int length = data.length();
@@ -193,7 +196,7 @@ public class RestMultiSearchAction extends BaseRestHandler {
             } else {
                 try (XContentParser requestParser = XContentFactory.xContent(slice).createParser(slice)) {
                     queryParseContext.reset(requestParser);
-                    searchRequest.source(SearchSourceBuilder.parseSearchSource(requestParser, queryParseContext, aggParsers));
+                    searchRequest.source(SearchSourceBuilder.parseSearchSource(requestParser, queryParseContext, aggParsers, suggesters));
                 }
             }
             // move pointers
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
index d1cd09373f7..9d533d15ff2 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -48,6 +48,8 @@ import org.elasticsearch.search.fetch.source.FetchSourceContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.search.suggest.SuggestBuilder;
+import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
+import org.elasticsearch.search.suggest.Suggesters;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -64,13 +66,15 @@ public class RestSearchAction extends BaseRestHandler {
 
     private final IndicesQueriesRegistry queryRegistry;
     private final AggregatorParsers aggParsers;
+    private final Suggesters suggesters;
 
     @Inject
     public RestSearchAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry queryRegistry,
-            AggregatorParsers aggParsers) {
+            AggregatorParsers aggParsers, Suggesters suggesters) {
         super(settings, client);
         this.queryRegistry = queryRegistry;
         this.aggParsers = aggParsers;
+        this.suggesters = suggesters;
         controller.registerHandler(GET, "/_search", this);
         controller.registerHandler(POST, "/_search", this);
         controller.registerHandler(GET, "/{index}/_search", this);
@@ -88,7 +92,7 @@ public class RestSearchAction extends BaseRestHandler {
     @Override
     public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException {
         SearchRequest searchRequest = new SearchRequest();
-        RestSearchAction.parseSearchRequest(searchRequest, queryRegistry, request, parseFieldMatcher, aggParsers, null);
+        parseSearchRequest(searchRequest, queryRegistry, request, parseFieldMatcher, aggParsers, suggesters, null);
         client.search(searchRequest, new RestStatusToXContentListener<>(channel));
     }
 
@@ -101,8 +105,10 @@ public class RestSearchAction extends BaseRestHandler {
      *            content is read from the request using
      *            RestAction.hasBodyContent.
      */
-    public static void parseSearchRequest(SearchRequest searchRequest, IndicesQueriesRegistry indicesQueriesRegistry, RestRequest request, 
-            ParseFieldMatcher parseFieldMatcher, AggregatorParsers aggParsers, BytesReference restContent) throws IOException {
+    public static void parseSearchRequest(SearchRequest searchRequest, IndicesQueriesRegistry indicesQueriesRegistry, RestRequest request,
+            ParseFieldMatcher parseFieldMatcher, AggregatorParsers aggParsers, Suggesters suggesters, BytesReference restContent)
+        throws IOException {
+
         if (searchRequest.source() == null) {
             searchRequest.source(new SearchSourceBuilder());
         }
@@ -117,16 +123,15 @@ public class RestSearchAction extends BaseRestHandler {
         }
         if (restContent != null) {
             QueryParseContext context = new QueryParseContext(indicesQueriesRegistry);
-            if (isTemplateRequest) {
-                try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
-                    context.reset(parser);
-                    context.parseFieldMatcher(parseFieldMatcher);
+            try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
+                context.reset(parser);
+                context.parseFieldMatcher(parseFieldMatcher);
+                if (isTemplateRequest) {
                     Template template = TemplateQueryParser.parse(parser, context.parseFieldMatcher(), "params", "template");
                     searchRequest.template(template);
+                } else {
+                    searchRequest.source().parseXContent(parser, context, aggParsers, suggesters);
                 }
-            } else {
-                RestActions.parseRestSearchSource(searchRequest.source(), restContent, indicesQueriesRegistry, parseFieldMatcher,
-                        aggParsers);
             }
         }
 
@@ -254,8 +259,10 @@ public class RestSearchAction extends BaseRestHandler {
             String suggestText = request.param("suggest_text", request.param("q"));
             int suggestSize = request.paramAsInt("suggest_size", 5);
             String suggestMode = request.param("suggest_mode");
-            searchSourceBuilder.suggest(new SuggestBuilder().addSuggestion(
-                    termSuggestion(suggestField).field(suggestField).text(suggestText).size(suggestSize).suggestMode(suggestMode)));
+            searchSourceBuilder.suggest(new SuggestBuilder().addSuggestion(suggestField,
+                    termSuggestion(suggestField)
+                        .text(suggestText).size(suggestSize)
+                        .suggestMode(SuggestMode.resolve(suggestMode))));
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/suggest/RestSuggestAction.java b/core/src/main/java/org/elasticsearch/rest/action/suggest/RestSuggestAction.java
index 4e6b88b68b8..1bbc662929c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/suggest/RestSuggestAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/suggest/RestSuggestAction.java
@@ -19,14 +19,20 @@
 
 package org.elasticsearch.rest.action.suggest;
 
-import org.elasticsearch.action.suggest.SuggestRequest;
-import org.elasticsearch.action.suggest.SuggestResponse;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
@@ -36,7 +42,12 @@ import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.support.RestActions;
 import org.elasticsearch.rest.action.support.RestBuilderListener;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.suggest.Suggest;
+import org.elasticsearch.search.suggest.SuggestBuilder;
+import org.elasticsearch.search.suggest.Suggesters;
+
+import java.io.IOException;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
@@ -47,9 +58,15 @@ import static org.elasticsearch.rest.action.support.RestActions.buildBroadcastSh
  */
 public class RestSuggestAction extends BaseRestHandler {
 
+    private final IndicesQueriesRegistry queryRegistry;
+    private final Suggesters suggesters;
+
     @Inject
-    public RestSuggestAction(Settings settings, RestController controller, Client client) {
+    public RestSuggestAction(Settings settings, RestController controller, Client client,
+                             IndicesQueriesRegistry queryRegistry, Suggesters suggesters) {
         super(settings, client);
+        this.queryRegistry = queryRegistry;
+        this.suggesters = suggesters;
         controller.registerHandler(POST, "/_suggest", this);
         controller.registerHandler(GET, "/_suggest", this);
         controller.registerHandler(POST, "/{index}/_suggest", this);
@@ -57,26 +74,33 @@ public class RestSuggestAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
-        SuggestRequest suggestRequest = new SuggestRequest(Strings.splitStringByCommaToArray(request.param("index")));
-        suggestRequest.indicesOptions(IndicesOptions.fromRequest(request, suggestRequest.indicesOptions()));
+    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException {
+        final SearchRequest searchRequest = new SearchRequest(Strings.splitStringByCommaToArray(request.param("index")), new SearchSourceBuilder());
+        searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions()));
         if (RestActions.hasBodyContent(request)) {
-            suggestRequest.suggest(RestActions.getRestContent(request));
+            final BytesReference sourceBytes = RestActions.getRestContent(request);
+            try (XContentParser parser = XContentFactory.xContent(sourceBytes).createParser(sourceBytes)) {
+                final QueryParseContext context = new QueryParseContext(queryRegistry);
+                context.reset(parser);
+                context.parseFieldMatcher(parseFieldMatcher);
+                searchRequest.source().suggest(SuggestBuilder.fromXContent(context, suggesters));
+            }
         } else {
             throw new IllegalArgumentException("no content or source provided to execute suggestion");
         }
-        suggestRequest.routing(request.param("routing"));
-        suggestRequest.preference(request.param("preference"));
-
-        client.suggest(suggestRequest, new RestBuilderListener<SuggestResponse>(channel) {
+        searchRequest.routing(request.param("routing"));
+        searchRequest.preference(request.param("preference"));
+        client.search(searchRequest, new RestBuilderListener<SearchResponse>(channel) {
             @Override
-            public RestResponse buildResponse(SuggestResponse response, XContentBuilder builder) throws Exception {
-                RestStatus restStatus = RestStatus.status(response.getSuccessfulShards(), response.getTotalShards(), response.getShardFailures());
+            public RestResponse buildResponse(SearchResponse response, XContentBuilder builder) throws Exception {
+                RestStatus restStatus = RestStatus.status(response.getSuccessfulShards(),
+                    response.getTotalShards(), response.getShardFailures());
                 builder.startObject();
-                buildBroadcastShardsHeader(builder, request, response);
+                buildBroadcastShardsHeader(builder, request, response.getTotalShards(),
+                    response.getSuccessfulShards(), response.getFailedShards(), response.getShardFailures());
                 Suggest suggest = response.getSuggest();
                 if (suggest != null) {
-                    suggest.toXContent(builder, request);
+                    suggest.toInnerXContent(builder, request);
                 }
                 builder.endObject();
                 return new BytesRestResponse(restStatus, builder);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java b/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java
index 692a9dc3402..55063664343 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java
@@ -42,6 +42,7 @@ import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.suggest.Suggesters;
 
 import java.io.IOException;
 
@@ -114,16 +115,6 @@ public class RestActions {
         return queryBuilder;
     }
 
-    public static void parseRestSearchSource(SearchSourceBuilder source, BytesReference sourceBytes, IndicesQueriesRegistry queryRegistry,
-            ParseFieldMatcher parseFieldMatcher, AggregatorParsers aggParsers)
-            throws IOException {
-        XContentParser parser = XContentFactory.xContent(sourceBytes).createParser(sourceBytes);
-        QueryParseContext queryParseContext = new QueryParseContext(queryRegistry);
-        queryParseContext.reset(parser);
-        queryParseContext.parseFieldMatcher(parseFieldMatcher);
-        source.parseXContent(parser, queryParseContext, aggParsers);
-    }
-
     /**
      * Get Rest content from either payload or source parameter
      * @param request Rest request
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java
index 8e1ac1c8d77..90c617540fc 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -46,6 +46,7 @@ import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
@@ -55,7 +56,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.query.TemplateQueryParser;
-import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.watcher.FileChangesListener;
 import org.elasticsearch.watcher.FileWatcher;
@@ -84,10 +84,13 @@ public class ScriptService extends AbstractComponent implements Closeable {
 
     static final String DISABLE_DYNAMIC_SCRIPTING_SETTING = "script.disable_dynamic";
 
-    public static final Setting<Integer> SCRIPT_CACHE_SIZE_SETTING = Setting.intSetting("script.cache.max_size", 100, 0, false, Setting.Scope.CLUSTER);
-    public static final Setting<TimeValue> SCRIPT_CACHE_EXPIRE_SETTING = Setting.positiveTimeSetting("script.cache.expire", TimeValue.timeValueMillis(0), false, Setting.Scope.CLUSTER);
+    public static final Setting<Integer> SCRIPT_CACHE_SIZE_SETTING =
+        Setting.intSetting("script.cache.max_size", 100, 0, Property.NodeScope);
+    public static final Setting<TimeValue> SCRIPT_CACHE_EXPIRE_SETTING =
+        Setting.positiveTimeSetting("script.cache.expire", TimeValue.timeValueMillis(0), Property.NodeScope);
     public static final String SCRIPT_INDEX = ".scripts";
-    public static final Setting<Boolean> SCRIPT_AUTO_RELOAD_ENABLED_SETTING = Setting.boolSetting("script.auto_reload_enabled", true, false, Setting.Scope.CLUSTER);
+    public static final Setting<Boolean> SCRIPT_AUTO_RELOAD_ENABLED_SETTING =
+        Setting.boolSetting("script.auto_reload_enabled", true, Property.NodeScope);
 
     private final String defaultLang;
 
@@ -225,6 +228,8 @@ public class ScriptService extends AbstractComponent implements Closeable {
         return scriptEngineService;
     }
 
+
+
     /**
      * Checks if a script can be executed and compiles it if needed, or returns the previously compiled and cached script.
      */
@@ -516,46 +521,53 @@ public class ScriptService extends AbstractComponent implements Closeable {
 
     private class ScriptChangesListener extends FileChangesListener {
 
-        private Tuple<String, String> scriptNameExt(Path file) {
+        private Tuple<String, String> getScriptNameExt(Path file) {
             Path scriptPath = scriptsDirectory.relativize(file);
             int extIndex = scriptPath.toString().lastIndexOf('.');
-            if (extIndex != -1) {
-                String ext = scriptPath.toString().substring(extIndex + 1);
-                String scriptName = scriptPath.toString().substring(0, extIndex).replace(scriptPath.getFileSystem().getSeparator(), "_");
-                return new Tuple<>(scriptName, ext);
-            } else {
+            if (extIndex <= 0) {
                 return null;
             }
+
+            String ext = scriptPath.toString().substring(extIndex + 1);
+            if (ext.isEmpty()) {
+                return null;
+            }
+
+            String scriptName = scriptPath.toString().substring(0, extIndex).replace(scriptPath.getFileSystem().getSeparator(), "_");
+            return new Tuple<>(scriptName, ext);
         }
 
         @Override
         public void onFileInit(Path file) {
+            Tuple<String, String> scriptNameExt = getScriptNameExt(file);
+            if (scriptNameExt == null) {
+                logger.debug("Skipped script with invalid extension : [{}]", file);
+                return;
+            }
             if (logger.isTraceEnabled()) {
                 logger.trace("Loading script file : [{}]", file);
             }
-            Tuple<String, String> scriptNameExt = scriptNameExt(file);
-            if (scriptNameExt != null) {
-                ScriptEngineService engineService = getScriptEngineServiceForFileExt(scriptNameExt.v2());
-                if (engineService == null) {
-                    logger.warn("no script engine found for [{}]", scriptNameExt.v2());
-                } else {
-                    try {
-                        //we don't know yet what the script will be used for, but if all of the operations for this lang
-                        // with file scripts are disabled, it makes no sense to even compile it and cache it.
-                        if (isAnyScriptContextEnabled(engineService.getTypes().get(0), engineService, ScriptType.FILE)) {
-                            logger.info("compiling script file [{}]", file.toAbsolutePath());
-                            try(InputStreamReader reader = new InputStreamReader(Files.newInputStream(file), StandardCharsets.UTF_8)) {
-                                String script = Streams.copyToString(reader);
-                                CacheKey cacheKey = new CacheKey(engineService, scriptNameExt.v1(), null, Collections.emptyMap());
-                                staticCache.put(cacheKey, new CompiledScript(ScriptType.FILE, scriptNameExt.v1(), engineService.getTypes().get(0), engineService.compile(script, Collections.emptyMap())));
-                                scriptMetrics.onCompilation();
-                            }
-                        } else {
-                            logger.warn("skipping compile of script file [{}] as all scripted operations are disabled for file scripts", file.toAbsolutePath());
+
+            ScriptEngineService engineService = getScriptEngineServiceForFileExt(scriptNameExt.v2());
+            if (engineService == null) {
+                logger.warn("No script engine found for [{}]", scriptNameExt.v2());
+            } else {
+                try {
+                    //we don't know yet what the script will be used for, but if all of the operations for this lang
+                    // with file scripts are disabled, it makes no sense to even compile it and cache it.
+                    if (isAnyScriptContextEnabled(engineService.getTypes().get(0), engineService, ScriptType.FILE)) {
+                        logger.info("compiling script file [{}]", file.toAbsolutePath());
+                        try (InputStreamReader reader = new InputStreamReader(Files.newInputStream(file), StandardCharsets.UTF_8)) {
+                            String script = Streams.copyToString(reader);
+                            CacheKey cacheKey = new CacheKey(engineService, scriptNameExt.v1(), null, Collections.emptyMap());
+                            staticCache.put(cacheKey, new CompiledScript(ScriptType.FILE, scriptNameExt.v1(), engineService.getTypes().get(0), engineService.compile(script, Collections.emptyMap())));
+                            scriptMetrics.onCompilation();
                         }
-                    } catch (Throwable e) {
-                        logger.warn("failed to load/compile script [{}]", e, scriptNameExt.v1());
+                    } else {
+                        logger.warn("skipping compile of script file [{}] as all scripted operations are disabled for file scripts", file.toAbsolutePath());
                     }
+                } catch (Throwable e) {
+                    logger.warn("failed to load/compile script [{}]", e, scriptNameExt.v1());
                 }
             }
         }
@@ -567,7 +579,7 @@ public class ScriptService extends AbstractComponent implements Closeable {
 
         @Override
         public void onFileDeleted(Path file) {
-            Tuple<String, String> scriptNameExt = scriptNameExt(file);
+            Tuple<String, String> scriptNameExt = getScriptNameExt(file);
             if (scriptNameExt != null) {
                 ScriptEngineService engineService = getScriptEngineServiceForFileExt(scriptNameExt.v2());
                 assert engineService != null;
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
index 8ececfe25bb..1bf7fdfc843 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
@@ -21,6 +21,7 @@ package org.elasticsearch.script;
 
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 import java.util.ArrayList;
@@ -44,8 +45,7 @@ public class ScriptSettings {
                 ScriptModes.sourceKey(scriptType),
                 scriptType.getDefaultScriptMode().getMode(),
                 ScriptMode::parse,
-                false,
-                Setting.Scope.CLUSTER));
+                Property.NodeScope));
         }
         SCRIPT_TYPE_SETTING_MAP = Collections.unmodifiableMap(scriptTypeSettingMap);
     }
@@ -66,7 +66,7 @@ public class ScriptSettings {
                 throw new IllegalArgumentException("unregistered default language [" + setting + "]");
             }
             return setting;
-        }, false, Setting.Scope.CLUSTER);
+        }, Property.NodeScope);
     }
 
     private static Map<ScriptContext, Setting<ScriptMode>> contextSettings(ScriptContextRegistry scriptContextRegistry) {
@@ -76,8 +76,7 @@ public class ScriptSettings {
                 ScriptModes.operationKey(scriptContext),
                 ScriptMode.OFF.getMode(),
                 ScriptMode::parse,
-                false,
-                Setting.Scope.CLUSTER
+                Property.NodeScope
             ));
         }
         return scriptContextSettingMap;
@@ -137,8 +136,7 @@ public class ScriptSettings {
                                 ScriptModes.getKey(language, scriptType, scriptContext),
                                 defaultSetting,
                                 ScriptMode::parse,
-                                false,
-                                Setting.Scope.CLUSTER);
+                                Property.NodeScope);
                         scriptModeSettings.add(setting);
                     }
                 }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchException.java b/core/src/main/java/org/elasticsearch/search/SearchException.java
index 0d181cc1dce..535f8acd446 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchException.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchException.java
@@ -45,7 +45,7 @@ public class SearchException extends ElasticsearchException implements Elasticse
     public SearchException(StreamInput in) throws IOException {
         super(in);
         if (in.readBoolean()) {
-            shardTarget = SearchShardTarget.readSearchShardTarget(in);
+            shardTarget = new SearchShardTarget(in);
         } else {
             shardTarget = null;
         }
@@ -54,7 +54,12 @@ public class SearchException extends ElasticsearchException implements Elasticse
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeOptionalStreamable(shardTarget);
+        if (shardTarget == null) {
+            out.writeBoolean(false);
+        } else {
+            out.writeBoolean(true);
+            shardTarget.writeTo(out);
+        }
     }
 
     public SearchShardTarget shard() {
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index 0d4faca3672..8756a31c444 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -21,21 +21,13 @@ package org.elasticsearch.search;
 
 import org.apache.lucene.search.BooleanQuery;
 import org.elasticsearch.common.geo.ShapesAvailability;
-import org.elasticsearch.common.geo.builders.CircleBuilder;
-import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
-import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder;
-import org.elasticsearch.common.geo.builders.LineStringBuilder;
-import org.elasticsearch.common.geo.builders.MultiLineStringBuilder;
-import org.elasticsearch.common.geo.builders.MultiPointBuilder;
-import org.elasticsearch.common.geo.builders.MultiPolygonBuilder;
-import org.elasticsearch.common.geo.builders.PointBuilder;
-import org.elasticsearch.common.geo.builders.PolygonBuilder;
-import org.elasticsearch.common.geo.builders.ShapeBuilder;
+import org.elasticsearch.common.geo.builders.ShapeBuilders;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.percolator.PercolatorHighlightSubFetchPhase;
 import org.elasticsearch.index.query.BoolQueryParser;
 import org.elasticsearch.index.query.BoostingQueryParser;
 import org.elasticsearch.index.query.CommonTermsQueryParser;
@@ -62,6 +54,7 @@ import org.elasticsearch.index.query.MoreLikeThisQueryParser;
 import org.elasticsearch.index.query.MultiMatchQueryParser;
 import org.elasticsearch.index.query.NestedQueryParser;
 import org.elasticsearch.index.query.ParentIdQueryParser;
+import org.elasticsearch.index.query.PercolatorQueryParser;
 import org.elasticsearch.index.query.PrefixQueryParser;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParser;
@@ -224,8 +217,21 @@ import org.elasticsearch.search.highlight.Highlighters;
 import org.elasticsearch.search.query.QueryPhase;
 import org.elasticsearch.search.rescore.QueryRescorerBuilder;
 import org.elasticsearch.search.rescore.RescoreBuilder;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
+import org.elasticsearch.search.sort.ScoreSortBuilder;
+import org.elasticsearch.search.sort.ScriptSortBuilder;
+import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.suggest.Suggester;
 import org.elasticsearch.search.suggest.Suggesters;
+import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
+import org.elasticsearch.search.suggest.phrase.Laplace;
+import org.elasticsearch.search.suggest.phrase.LinearInterpolation;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
+import org.elasticsearch.search.suggest.phrase.SmoothingModel;
+import org.elasticsearch.search.suggest.phrase.StupidBackoff;
+import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -271,14 +277,17 @@ public class SearchModule extends AbstractModule {
 
         registerBuiltinFunctionScoreParsers();
         registerBuiltinQueryParsers();
+        registerBuiltinRescorers();
+        registerBuiltinSorts();
     }
 
     public void registerHighlighter(String key, Class<? extends Highlighter> clazz) {
         highlighters.registerExtension(key, clazz);
     }
 
-    public void registerSuggester(String key, Class<? extends Suggester> suggester) {
-        suggesters.registerExtension(key, suggester);
+    public void registerSuggester(String key, Suggester<?> suggester) {
+        suggesters.registerExtension(key, suggester.getClass());
+        namedWriteableRegistry.registerPrototype(SuggestionBuilder.class, suggester.getBuilderPrototype());
     }
 
     /**
@@ -334,7 +343,6 @@ public class SearchModule extends AbstractModule {
         configureSuggesters();
         configureFetchSubPhase();
         configureShapes();
-        configureRescorers();
     }
 
     protected void configureFetchSubPhase() {
@@ -347,6 +355,7 @@ public class SearchModule extends AbstractModule {
         fetchSubPhaseMultibinder.addBinding().to(MatchedQueriesFetchSubPhase.class);
         fetchSubPhaseMultibinder.addBinding().to(HighlightPhase.class);
         fetchSubPhaseMultibinder.addBinding().to(ParentFieldSubFetchPhase.class);
+        fetchSubPhaseMultibinder.addBinding().to(PercolatorHighlightSubFetchPhase.class);
         for (Class<? extends FetchSubPhase> clazz : fetchSubPhases) {
             fetchSubPhaseMultibinder.addBinding().to(clazz);
         }
@@ -371,6 +380,12 @@ public class SearchModule extends AbstractModule {
 
     protected void configureSuggesters() {
         suggesters.bind(binder());
+        namedWriteableRegistry.registerPrototype(SuggestionBuilder.class, TermSuggestionBuilder.PROTOTYPE);
+        namedWriteableRegistry.registerPrototype(SuggestionBuilder.class, PhraseSuggestionBuilder.PROTOTYPE);
+        namedWriteableRegistry.registerPrototype(SuggestionBuilder.class, CompletionSuggestionBuilder.PROTOTYPE);
+        namedWriteableRegistry.registerPrototype(SmoothingModel.class, Laplace.PROTOTYPE);
+        namedWriteableRegistry.registerPrototype(SmoothingModel.class, LinearInterpolation.PROTOTYPE);
+        namedWriteableRegistry.registerPrototype(SmoothingModel.class, StupidBackoff.PROTOTYPE);
     }
 
     protected void configureHighlighters() {
@@ -455,20 +470,19 @@ public class SearchModule extends AbstractModule {
 
     private void configureShapes() {
         if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) {
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, PointBuilder.PROTOTYPE);
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, CircleBuilder.PROTOTYPE);
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, EnvelopeBuilder.PROTOTYPE);
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, MultiPointBuilder.PROTOTYPE);
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, LineStringBuilder.PROTOTYPE);
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, MultiLineStringBuilder.PROTOTYPE);
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, PolygonBuilder.PROTOTYPE);
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, MultiPolygonBuilder.PROTOTYPE);
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, GeometryCollectionBuilder.PROTOTYPE);
+            ShapeBuilders.register(namedWriteableRegistry);
         }
     }
 
-    private void configureRescorers() {
-        namedWriteableRegistry.registerPrototype(RescoreBuilder.class, QueryRescorerBuilder.PROTOTYPE);
+    private void registerBuiltinRescorers() {
+        namedWriteableRegistry.register(RescoreBuilder.class, QueryRescorerBuilder.NAME, QueryRescorerBuilder::new);
+    }
+
+    private void registerBuiltinSorts() {
+        namedWriteableRegistry.register(SortBuilder.class, GeoDistanceSortBuilder.NAME, GeoDistanceSortBuilder::new);
+        namedWriteableRegistry.register(SortBuilder.class, ScoreSortBuilder.NAME, ScoreSortBuilder::new);
+        namedWriteableRegistry.register(SortBuilder.class, ScriptSortBuilder.NAME, ScriptSortBuilder::new);
+        namedWriteableRegistry.register(SortBuilder.class, FieldSortBuilder.NAME, FieldSortBuilder::new);
     }
 
     private void registerBuiltinFunctionScoreParsers() {
@@ -531,6 +545,7 @@ public class SearchModule extends AbstractModule {
         registerQueryParser(ExistsQueryParser::new);
         registerQueryParser(MatchNoneQueryParser::new);
         registerQueryParser(ParentIdQueryParser::new);
+        registerQueryParser(PercolatorQueryParser::new);
         if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) {
             registerQueryParser(GeoShapeQueryParser::new);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index eb3568296ad..98085bfed35 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -21,11 +21,12 @@ package org.elasticsearch.search;
 
 import com.carrotsearch.hppc.ObjectFloatHashMap;
 import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.TopDocs;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cache.recycler.PageCacheRecycler;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -34,19 +35,20 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.ConcurrentMapLong;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.fieldstats.FieldStatsProvider;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.search.stats.ShardSearchStats;
@@ -54,7 +56,6 @@ import org.elasticsearch.index.search.stats.StatsGroupsParseElement;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.indices.IndicesRequestCache;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
@@ -91,12 +92,15 @@ import org.elasticsearch.search.query.QuerySearchResultProvider;
 import org.elasticsearch.search.query.ScrollQuerySearchResult;
 import org.elasticsearch.search.rescore.RescoreBuilder;
 import org.elasticsearch.search.searchafter.SearchAfterBuilder;
+import org.elasticsearch.search.sort.SortBuilder;
+import org.elasticsearch.search.suggest.Suggesters;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.atomic.AtomicLong;
@@ -111,11 +115,14 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes;
 public class SearchService extends AbstractLifecycleComponent<SearchService> implements IndexEventListener {
 
     // we can have 5 minutes here, since we make sure to clean with search requests and when shard/index closes
-    public static final Setting<TimeValue> DEFAULT_KEEPALIVE_SETTING = Setting.positiveTimeSetting("search.default_keep_alive", timeValueMinutes(5), false, Setting.Scope.CLUSTER);
-    public static final Setting<TimeValue> KEEPALIVE_INTERVAL_SETTING = Setting.positiveTimeSetting("search.keep_alive_interval", timeValueMinutes(1), false, Setting.Scope.CLUSTER);
+    public static final Setting<TimeValue> DEFAULT_KEEPALIVE_SETTING =
+        Setting.positiveTimeSetting("search.default_keep_alive", timeValueMinutes(5), Property.NodeScope);
+    public static final Setting<TimeValue> KEEPALIVE_INTERVAL_SETTING =
+        Setting.positiveTimeSetting("search.keep_alive_interval", timeValueMinutes(1), Property.NodeScope);
 
     public static final TimeValue NO_TIMEOUT = timeValueMillis(-1);
-    public static final Setting<TimeValue> DEFAULT_SEARCH_TIMEOUT_SETTING = Setting.timeSetting("search.default_search_timeout", NO_TIMEOUT, true, Setting.Scope.CLUSTER);
+    public static final Setting<TimeValue> DEFAULT_SEARCH_TIMEOUT_SETTING =
+        Setting.timeSetting("search.default_search_timeout", NO_TIMEOUT, Property.Dynamic, Property.NodeScope);
 
 
     private final ThreadPool threadPool;
@@ -149,14 +156,16 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
     private final Map<String, SearchParseElement> elementParsers;
 
     private final ParseFieldMatcher parseFieldMatcher;
-    private AggregatorParsers aggParsers;
+    private final AggregatorParsers aggParsers;
+    private final Suggesters suggesters;
 
     @Inject
     public SearchService(Settings settings, ClusterSettings clusterSettings, ClusterService clusterService, IndicesService indicesService,
-            ThreadPool threadPool, ScriptService scriptService, PageCacheRecycler pageCacheRecycler, BigArrays bigArrays, DfsPhase dfsPhase,
-            QueryPhase queryPhase, FetchPhase fetchPhase, AggregatorParsers aggParsers) {
+                         ThreadPool threadPool, ScriptService scriptService, PageCacheRecycler pageCacheRecycler, BigArrays bigArrays, DfsPhase dfsPhase,
+                         QueryPhase queryPhase, FetchPhase fetchPhase, AggregatorParsers aggParsers, Suggesters suggesters) {
         super(settings);
         this.aggParsers = aggParsers;
+        this.suggesters = suggesters;
         this.parseFieldMatcher = new ParseFieldMatcher(settings);
         this.threadPool = threadPool;
         this.clusterService = clusterService;
@@ -192,7 +201,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
     public void afterIndexClosed(Index index, Settings indexSettings) {
         // once an index is closed we can just clean up all the pending search context information
         // to release memory and let references to the filesystem go etc.
-        IndexMetaData idxMeta = SearchService.this.clusterService.state().metaData().index(index.getName());
+        IndexMetaData idxMeta = SearchService.this.clusterService.state().metaData().index(index);
         if (idxMeta != null && idxMeta.getState() == IndexMetaData.State.CLOSE) {
             // we need to check if it's really closed
             // since sometimes due to a relocation we already closed the shard and that causes the index to be closed
@@ -234,7 +243,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
         FutureUtils.cancel(keepAliveReaper);
     }
 
-    public DfsSearchResult executeDfsPhase(ShardSearchRequest request) {
+    public DfsSearchResult executeDfsPhase(ShardSearchRequest request) throws IOException {
         final SearchContext context = createAndPutContext(request);
         try {
             contextProcessing(context);
@@ -263,7 +272,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
         }
     }
 
-    public QuerySearchResultProvider executeQueryPhase(ShardSearchRequest request) {
+    public QuerySearchResultProvider executeQueryPhase(ShardSearchRequest request) throws IOException {
         final SearchContext context = createAndPutContext(request);
         final ShardSearchStats shardSearchStats = context.indexShard().searchService();
         try {
@@ -355,7 +364,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
         }
     }
 
-    public QueryFetchSearchResult executeFetchPhase(ShardSearchRequest request) {
+    public QueryFetchSearchResult executeFetchPhase(ShardSearchRequest request) throws IOException {
         final SearchContext context = createAndPutContext(request);
         contextProcessing(context);
         try {
@@ -512,7 +521,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
         return context;
     }
 
-    final SearchContext createAndPutContext(ShardSearchRequest request) {
+    final SearchContext createAndPutContext(ShardSearchRequest request) throws IOException {
         SearchContext context = createContext(request, null);
         boolean success = false;
         try {
@@ -530,11 +539,10 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
         }
     }
 
-    final SearchContext createContext(ShardSearchRequest request, @Nullable Engine.Searcher searcher) {
-        IndexService indexService = indicesService.indexServiceSafe(request.index());
-        IndexShard indexShard = indexService.getShard(request.shardId());
-
-        SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().id(), indexShard.shardId().getIndex(), request.shardId());
+    final SearchContext createContext(ShardSearchRequest request, @Nullable Engine.Searcher searcher) throws IOException {
+        IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
+        IndexShard indexShard = indexService.getShard(request.shardId().getId());
+        SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().id(), indexShard.shardId());
 
         Engine.Searcher engineSearcher = searcher == null ? indexShard.acquireSearcher("search") : searcher;
 
@@ -542,7 +550,15 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
                 indexService,
                 indexShard, scriptService, pageCacheRecycler, bigArrays, threadPool.estimatedTimeInMillisCounter(), parseFieldMatcher,
                 defaultSearchTimeout, fetchPhase);
+        context.getQueryShardContext().setFieldStatsProvider(new FieldStatsProvider(engineSearcher, indexService.mapperService()));
         SearchContext.setCurrent(context);
+        request.rewrite(context.getQueryShardContext());
+        // reset that we have used nowInMillis from the context since it may
+        // have been rewritten so its no longer in the query and the request can
+        // be cached. If it is still present in the request (e.g. in a range
+        // aggregation) it will still be caught when the aggregation is
+        // evaluated.
+        context.resetNowInMillisUsed();
         try {
             if (request.scroll() != null) {
                 context.scrollContext(new ScrollContext());
@@ -555,7 +571,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
                     QueryParseContext queryParseContext = new QueryParseContext(indicesService.getIndicesQueryRegistry());
                     queryParseContext.reset(parser);
                     queryParseContext.parseFieldMatcher(parseFieldMatcher);
-                    parseSource(context, SearchSourceBuilder.parseSearchSource(parser, queryParseContext, aggParsers));
+                    parseSource(context, SearchSourceBuilder.parseSearchSource(parser, queryParseContext, aggParsers, suggesters));
                 }
             }
             parseSource(context, request.source());
@@ -669,33 +685,13 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
             context.parsedPostFilter(queryShardContext.toQuery(source.postFilter()));
         }
         if (source.sorts() != null) {
-            XContentParser completeSortParser = null;
             try {
-                XContentBuilder completeSortBuilder = XContentFactory.jsonBuilder();
-                completeSortBuilder.startObject();
-                completeSortBuilder.startArray("sort");
-                for (BytesReference sort : source.sorts()) {
-                    XContentParser parser = XContentFactory.xContent(sort).createParser(sort);
-                    parser.nextToken();
-                    completeSortBuilder.copyCurrentStructure(parser);
+                Optional<Sort> optionalSort = SortBuilder.buildSort(source.sorts(), context.getQueryShardContext());
+                if (optionalSort.isPresent()) {
+                    context.sort(optionalSort.get());
                 }
-                completeSortBuilder.endArray();
-                completeSortBuilder.endObject();
-                BytesReference completeSortBytes = completeSortBuilder.bytes();
-                completeSortParser = XContentFactory.xContent(completeSortBytes).createParser(completeSortBytes);
-                completeSortParser.nextToken();
-                completeSortParser.nextToken();
-                completeSortParser.nextToken();
-                this.elementParsers.get("sort").parse(completeSortParser, context);
-            } catch (Exception e) {
-                String sSource = "_na_";
-                try {
-                    sSource = source.toString();
-                } catch (Throwable e1) {
-                    // ignore
-                }
-                XContentLocation location = completeSortParser != null ? completeSortParser.getTokenLocation() : null;
-                throw new SearchParseException(context, "failed to parse sort source [" + sSource + "]", location, e);
+            } catch (IOException e) {
+                throw new SearchContextException(context, "failed to create sort elements", e);
             }
         }
         context.trackScores(source.trackScores());
@@ -718,26 +714,16 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
             }
         }
         if (source.suggest() != null) {
-            XContentParser suggestParser = null;
             try {
-                suggestParser = XContentFactory.xContent(source.suggest()).createParser(source.suggest());
-                suggestParser.nextToken();
-                this.elementParsers.get("suggest").parse(suggestParser, context);
-            } catch (Exception e) {
-                String sSource = "_na_";
-                try {
-                    sSource = source.toString();
-                } catch (Throwable e1) {
-                    // ignore
-                }
-                XContentLocation location = suggestParser != null ? suggestParser.getTokenLocation() : null;
-                throw new SearchParseException(context, "failed to parse suggest source [" + sSource + "]", location, e);
+                context.suggest(source.suggest().build(queryShardContext));
+            } catch (IOException e) {
+                throw new SearchContextException(context, "failed to create SuggestionSearchContext", e);
             }
         }
         if (source.rescores() != null) {
             try {
                 for (RescoreBuilder<?> rescore : source.rescores()) {
-                    context.addRescore(rescore.build(context.getQueryShardContext()));
+                    context.addRescore(rescore.build(queryShardContext));
                 }
             } catch (IOException e) {
                 throw new SearchContextException(context, "failed to create RescoreSearchContext", e);
@@ -762,7 +748,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
         if (source.highlighter() != null) {
             HighlightBuilder highlightBuilder = source.highlighter();
             try {
-                context.highlight(highlightBuilder.build(context.getQueryShardContext()));
+                context.highlight(highlightBuilder.build(queryShardContext));
             } catch (IOException e) {
                 throw new SearchContextException(context, "failed to create SearchContextHighlighter", e);
             }
@@ -802,6 +788,11 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
                     } else {
                         SearchParseElement parseElement = this.elementParsers.get(currentFieldName);
                         if (parseElement == null) {
+                            if (currentFieldName != null && currentFieldName.equals("suggest")) {
+                                throw new SearchParseException(context,
+                                    "suggest is not supported in [ext], please use SearchSourceBuilder#suggest(SuggestBuilder) instead",
+                                    extParser.getTokenLocation());
+                            }
                             throw new SearchParseException(context, "Unknown element [" + currentFieldName + "] in [ext]",
                                     extParser.getTokenLocation());
                         } else {
diff --git a/core/src/main/java/org/elasticsearch/search/SearchShardTarget.java b/core/src/main/java/org/elasticsearch/search/SearchShardTarget.java
index d3958505d70..d675a93b691 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchShardTarget.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchShardTarget.java
@@ -23,28 +23,38 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
 
 import java.io.IOException;
 
 /**
  * The target that the search request was executed on.
  */
-public class SearchShardTarget implements Streamable, Comparable<SearchShardTarget> {
+public class SearchShardTarget implements Writeable<SearchShardTarget>, Comparable<SearchShardTarget> {
 
     private Text nodeId;
     private Text index;
-    private int shardId;
+    private ShardId shardId;
 
-    private SearchShardTarget() {
+    public SearchShardTarget(StreamInput in) throws IOException {
+        if (in.readBoolean()) {
+            nodeId = in.readText();
+        }
+        shardId = ShardId.readShardId(in);
+        index = new Text(shardId.getIndexName());
+    }
 
+    public SearchShardTarget(String nodeId, ShardId shardId) {
+        this.nodeId = nodeId == null ? null : new Text(nodeId);
+        this.index = new Text(shardId.getIndexName());
+        this.shardId = shardId;
     }
 
     public SearchShardTarget(String nodeId, Index index, int shardId) {
-        this.nodeId = nodeId == null ? null : new Text(nodeId);
-        this.index = new Text(index.getName());
-        this.shardId = shardId;
+        this(nodeId,  new ShardId(index, shardId));
     }
 
     @Nullable
@@ -73,36 +83,26 @@ public class SearchShardTarget implements Streamable, Comparable<SearchShardTarg
         return this.index;
     }
 
-    public int shardId() {
+    public ShardId shardId() {
         return shardId;
     }
 
-    public int getShardId() {
+    public ShardId getShardId() {
         return shardId;
     }
 
-    public static SearchShardTarget readSearchShardTarget(StreamInput in) throws IOException {
-        SearchShardTarget result = new SearchShardTarget();
-        result.readFrom(in);
-        return result;
-    }
-
     @Override
     public int compareTo(SearchShardTarget o) {
         int i = index.string().compareTo(o.index());
         if (i == 0) {
-            i = shardId - o.shardId;
+            i = shardId.getId() - o.shardId.id();
         }
         return i;
     }
 
     @Override
-    public void readFrom(StreamInput in) throws IOException {
-        if (in.readBoolean()) {
-            nodeId = in.readText();
-        }
-        index = in.readText();
-        shardId = in.readVInt();
+    public SearchShardTarget readFrom(StreamInput in) throws IOException {
+        return new SearchShardTarget(in);
     }
 
     @Override
@@ -113,19 +113,15 @@ public class SearchShardTarget implements Streamable, Comparable<SearchShardTarg
             out.writeBoolean(true);
             out.writeText(nodeId);
         }
-        out.writeText(index);
-        out.writeVInt(shardId);
+        shardId.writeTo(out);
     }
 
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-
         SearchShardTarget that = (SearchShardTarget) o;
-
-        if (shardId != that.shardId) return false;
-        if (index != null ? !index.equals(that.index) : that.index != null) return false;
+        if (shardId.equals(that.shardId) == false) return false;
         if (nodeId != null ? !nodeId.equals(that.nodeId) : that.nodeId != null) return false;
 
         return true;
@@ -135,7 +131,7 @@ public class SearchShardTarget implements Streamable, Comparable<SearchShardTarg
     public int hashCode() {
         int result = nodeId != null ? nodeId.hashCode() : 0;
         result = 31 * result + (index != null ? index.hashCode() : 0);
-        result = 31 * result + shardId;
+        result = 31 * result + shardId.hashCode();
         return result;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregator.java
index 17a7ae4ebe5..5aff086d23e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregator.java
@@ -194,6 +194,12 @@ public class FiltersAggregator extends BucketsAggregator {
             InternalFilters.InternalBucket bucket = new InternalFilters.InternalBucket(keys[i], 0, subAggs, keyed);
             buckets.add(bucket);
         }
+
+        if (showOtherBucket) {
+            InternalFilters.InternalBucket bucket = new InternalFilters.InternalBucket(otherBucketKey, 0, subAggs, keyed);
+            buckets.add(bucket);
+        }
+
         return new InternalFilters(name, buckets, keyed, pipelineAggregators(), metaData());
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregatorBuilder.java
index 5969265f75f..a11bfd113bb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregatorBuilder.java
@@ -211,7 +211,7 @@ public class GeoDistanceAggregatorBuilder extends ValuesSourceAggregatorBuilder<
         }
         factory.keyed = in.readBoolean();
         factory.distanceType = GeoDistance.readGeoDistanceFrom(in);
-        factory.unit = DistanceUnit.readDistanceUnit(in);
+        factory.unit = DistanceUnit.readFromStream(in);
         return factory;
     }
 
@@ -225,7 +225,7 @@ public class GeoDistanceAggregatorBuilder extends ValuesSourceAggregatorBuilder<
         }
         out.writeBoolean(keyed);
         distanceType.writeTo(out);
-        DistanceUnit.writeDistanceUnit(out, unit);
+        unit.writeTo(out);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregatorBuilder.java
index cd43777959e..de66f68103f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregatorBuilder.java
@@ -48,7 +48,7 @@ public class PercentileRanksAggregatorBuilder extends LeafOnly<ValuesSource.Nume
     private PercentilesMethod method = PercentilesMethod.TDIGEST;
     private int numberOfSignificantValueDigits = 3;
     private double compression = 100.0;
-    private boolean keyed = false;
+    private boolean keyed = true;
 
     public PercentileRanksAggregatorBuilder(String name) {
         super(name, InternalTDigestPercentileRanks.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregatorBuilder.java
index 302f2c1947c..4ab26948df3 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregatorBuilder.java
@@ -48,7 +48,7 @@ public class PercentilesAggregatorBuilder extends LeafOnly<ValuesSource.Numeric,
     private PercentilesMethod method = PercentilesMethod.TDIGEST;
     private int numberOfSignificantValueDigits = 3;
     private double compression = 100.0;
-    private boolean keyed = false;
+    private boolean keyed = true;
 
     public PercentilesAggregatorBuilder(String name) {
         super(name, InternalTDigestPercentiles.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java
index 9f5f3e443ef..fd60fa93857 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java
@@ -21,13 +21,9 @@ package org.elasticsearch.search.aggregations.metrics.tophits;
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregationInitializationException;
 import org.elasticsearch.search.aggregations.AggregatorBuilder;
@@ -38,6 +34,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField;
 import org.elasticsearch.search.fetch.source.FetchSourceContext;
 import org.elasticsearch.search.highlight.HighlightBuilder;
+import org.elasticsearch.search.sort.ScoreSortBuilder;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
@@ -57,7 +54,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
     private boolean explain = false;
     private boolean version = false;
     private boolean trackScores = false;
-    private List<BytesReference> sorts = null;
+    private List<SortBuilder<?>> sorts = null;
     private HighlightBuilder highlightBuilder;
     private List<String> fieldNames;
     private List<String> fieldDataFields;
@@ -119,6 +116,9 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
         if (order == null) {
             throw new IllegalArgumentException("sort [order] must not be null: [" + name + "]");
         }
+        if (name.equals(ScoreSortBuilder.NAME)) {
+            sort(SortBuilders.scoreSort().order(order));
+        }
         sort(SortBuilders.fieldSort(name).order(order));
         return this;
     }
@@ -133,6 +133,9 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
         if (name == null) {
             throw new IllegalArgumentException("sort [name] must not be null: [" + name + "]");
         }
+        if (name.equals(ScoreSortBuilder.NAME)) {
+            sort(SortBuilders.scoreSort());
+        }
         sort(SortBuilders.fieldSort(name));
         return this;
     }
@@ -140,39 +143,28 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
     /**
      * Adds a sort builder.
      */
-    public TopHitsAggregatorBuilder sort(SortBuilder sort) {
+    public TopHitsAggregatorBuilder sort(SortBuilder<?> sort) {
         if (sort == null) {
             throw new IllegalArgumentException("[sort] must not be null: [" + name + "]");
         }
-        try {
-            if (sorts == null) {
+        if (sorts == null) {
                 sorts = new ArrayList<>();
-            }
-            // NORELEASE when sort has been refactored and made writeable
-            // add the sortBuilcer to the List directly instead of
-            // serialising to XContent
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            sort.toXContent(builder, EMPTY_PARAMS);
-            builder.endObject();
-            sorts.add(builder.bytes());
-        } catch (IOException e) {
-            throw new RuntimeException(e);
         }
+        sorts.add(sort);
         return this;
     }
 
     /**
      * Adds a sort builder.
      */
-    public TopHitsAggregatorBuilder sorts(List<BytesReference> sorts) {
+    public TopHitsAggregatorBuilder sorts(List<SortBuilder<?>> sorts) {
         if (sorts == null) {
             throw new IllegalArgumentException("[sorts] must not be null: [" + name + "]");
         }
         if (this.sorts == null) {
             this.sorts = new ArrayList<>();
         }
-        for (BytesReference sort : sorts) {
+        for (SortBuilder<?> sort : sorts) {
             this.sorts.add(sort);
         }
         return this;
@@ -181,7 +173,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
     /**
      * Gets the bytes representing the sort builders for this request.
      */
-    public List<BytesReference> sorts() {
+    public List<SortBuilder<?>> sorts() {
         return sorts;
     }
 
@@ -509,10 +501,8 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
         }
         if (sorts != null) {
             builder.startArray(SearchSourceBuilder.SORT_FIELD.getPreferredName());
-            for (BytesReference sort : sorts) {
-                XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(sort);
-                parser.nextToken();
-                builder.copyCurrentStructure(parser);
+            for (SortBuilder<?> sort : sorts) {
+                    sort.toXContent(builder, params);
             }
             builder.endArray();
         }
@@ -562,9 +552,9 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
         factory.size = in.readVInt();
         if (in.readBoolean()) {
             int size = in.readVInt();
-            List<BytesReference> sorts = new ArrayList<>();
+            List<SortBuilder<?>> sorts = new ArrayList<>();
             for (int i = 0; i < size; i++) {
-                sorts.add(in.readBytesReference());
+                sorts.add(in.readSortBuilder());
             }
             factory.sorts = sorts;
         }
@@ -612,8 +602,8 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
         out.writeBoolean(hasSorts);
         if (hasSorts) {
             out.writeVInt(sorts.size());
-            for (BytesReference sort : sorts) {
-                out.writeBytesReference(sort);
+            for (SortBuilder<?> sort : sorts) {
+                out.writeSortBuilder(sort);
             }
         }
         out.writeBoolean(trackScores);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java
index c4d2165e308..be10f4ce878 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java
@@ -19,12 +19,7 @@
 
 package org.elasticsearch.search.aggregations.metrics.tophits;
 
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.common.xcontent.XContentParser;
+import org.apache.lucene.search.Sort;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.aggregations.Aggregator;
@@ -35,27 +30,27 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField;
 import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext;
-import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext.FieldDataField;
+import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.source.FetchSourceContext;
 import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.internal.SubSearchContext;
-import org.elasticsearch.search.sort.SortParseElement;
+import org.elasticsearch.search.sort.SortBuilder;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 public class TopHitsAggregatorFactory extends AggregatorFactory<TopHitsAggregatorFactory> {
 
-    private static final SortParseElement sortParseElement = new SortParseElement();
     private final int from;
     private final int size;
     private final boolean explain;
     private final boolean version;
     private final boolean trackScores;
-    private final List<BytesReference> sorts;
+    private final List<SortBuilder<?>> sorts;
     private final HighlightBuilder highlightBuilder;
     private final List<String> fieldNames;
     private final List<String> fieldDataFields;
@@ -63,7 +58,7 @@ public class TopHitsAggregatorFactory extends AggregatorFactory<TopHitsAggregato
     private final FetchSourceContext fetchSourceContext;
 
     public TopHitsAggregatorFactory(String name, Type type, int from, int size, boolean explain, boolean version, boolean trackScores,
-            List<BytesReference> sorts, HighlightBuilder highlightBuilder, List<String> fieldNames, List<String> fieldDataFields,
+            List<SortBuilder<?>> sorts, HighlightBuilder highlightBuilder, List<String> fieldNames, List<String> fieldDataFields,
             List<ScriptField> scriptFields, FetchSourceContext fetchSourceContext, AggregationContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactories, Map<String, Object> metaData) throws IOException {
         super(name, type, context, parent, subFactories, metaData);
@@ -90,27 +85,9 @@ public class TopHitsAggregatorFactory extends AggregatorFactory<TopHitsAggregato
         subSearchContext.from(from);
         subSearchContext.size(size);
         if (sorts != null) {
-            XContentParser completeSortParser = null;
-            try {
-                XContentBuilder completeSortBuilder = XContentFactory.jsonBuilder();
-                completeSortBuilder.startObject();
-                completeSortBuilder.startArray("sort");
-                for (BytesReference sort : sorts) {
-                    XContentParser parser = XContentFactory.xContent(sort).createParser(sort);
-                    parser.nextToken();
-                    completeSortBuilder.copyCurrentStructure(parser);
-                }
-                completeSortBuilder.endArray();
-                completeSortBuilder.endObject();
-                BytesReference completeSortBytes = completeSortBuilder.bytes();
-                completeSortParser = XContentFactory.xContent(completeSortBytes).createParser(completeSortBytes);
-                completeSortParser.nextToken();
-                completeSortParser.nextToken();
-                completeSortParser.nextToken();
-                sortParseElement.parse(completeSortParser, subSearchContext);
-            } catch (Exception e) {
-                XContentLocation location = completeSortParser != null ? completeSortParser.getTokenLocation() : null;
-                throw new ParsingException(location, "failed to parse sort source in aggregation [" + name + "]", e);
+            Optional<Sort> optionalSort = SortBuilder.buildSort(sorts, subSearchContext.getQueryShardContext());
+            if (optionalSort.isPresent()) {
+                subSearchContext.sort(optionalSort.get());
             }
         }
         if (fieldNames != null) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsParser.java
index 03921d620ee..9ec6b46dafe 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsParser.java
@@ -19,9 +19,6 @@
 package org.elasticsearch.search.aggregations.metrics.tophits;
 
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
@@ -30,6 +27,8 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField;
 import org.elasticsearch.search.fetch.source.FetchSourceContext;
 import org.elasticsearch.search.highlight.HighlightBuilder;
+import org.elasticsearch.search.sort.SortBuilder;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -124,9 +123,7 @@ public class TopHitsParser implements Aggregator.Parser {
                 } else if (context.parseFieldMatcher().match(currentFieldName, SearchSourceBuilder.HIGHLIGHT_FIELD)) {
                     factory.highlighter(HighlightBuilder.PROTOTYPE.fromXContent(context));
                 } else if (context.parseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SORT_FIELD)) {
-                    List<BytesReference> sorts = new ArrayList<>();
-                    XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser);
-                    sorts.add(xContentBuilder.bytes());
+                    List<SortBuilder<?>> sorts = SortBuilder.fromXContent(context);
                     factory.sorts(sorts);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
@@ -157,11 +154,7 @@ public class TopHitsParser implements Aggregator.Parser {
                     }
                     factory.fieldDataFields(fieldDataFields);
                 } else if (context.parseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SORT_FIELD)) {
-                    List<BytesReference> sorts = new ArrayList<>();
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser);
-                        sorts.add(xContentBuilder.bytes());
-                    }
+                    List<SortBuilder<?>> sorts = SortBuilder.fromXContent(context);
                     factory.sorts(sorts);
                 } else if (context.parseFieldMatcher().match(currentFieldName, SearchSourceBuilder._SOURCE_FIELD)) {
                     factory.fetchSource(FetchSourceContext.parse(parser, context));
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregator.java
index 1289da661c2..cea99cf868e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregator.java
@@ -92,7 +92,7 @@ public class DerivativePipelineAggregator extends PipelineAggregator {
         for (InternalHistogram.Bucket bucket : buckets) {
             Long thisBucketKey = resolveBucketKeyAsLong(bucket);
             Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], gapPolicy);
-            if (lastBucketValue != null) {
+            if (lastBucketValue != null && thisBucketValue != null) {
                 double gradient = thisBucketValue - lastBucketValue;
                 double xDiff = -1;
                 if (xAxisUnits != null) {
diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index 0cb83dbd2f9..c83794ced90 100644
--- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -40,6 +40,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregatorBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
@@ -51,10 +52,12 @@ import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.rescore.RescoreBuilder;
 import org.elasticsearch.search.searchafter.SearchAfterBuilder;
+import org.elasticsearch.search.sort.ScoreSortBuilder;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.search.suggest.SuggestBuilder;
+import org.elasticsearch.search.suggest.Suggesters;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -105,9 +108,10 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         return PROTOTYPE.readFrom(in);
     }
 
-    public static SearchSourceBuilder parseSearchSource(XContentParser parser, QueryParseContext context, AggregatorParsers aggParsers)
+    public static SearchSourceBuilder parseSearchSource(XContentParser parser, QueryParseContext context,
+                                                        AggregatorParsers aggParsers, Suggesters suggesters)
             throws IOException {
-        return PROTOTYPE.fromXContent(parser, context, aggParsers);
+        return PROTOTYPE.fromXContent(parser, context, aggParsers, suggesters);
     }
 
     /**
@@ -136,7 +140,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
 
     private Boolean version;
 
-    private List<BytesReference> sorts;
+    private List<SortBuilder<?>> sorts;
 
     private boolean trackScores = false;
 
@@ -156,7 +160,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
 
     private HighlightBuilder highlightBuilder;
 
-    private BytesReference suggestBuilder;
+    private SuggestBuilder suggestBuilder;
 
     private BytesReference innerHitsBuilder;
 
@@ -333,6 +337,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
      *            The sort ordering
      */
     public SearchSourceBuilder sort(String name, SortOrder order) {
+        if (name.equals(ScoreSortBuilder.NAME)) {
+            return sort(SortBuilders.scoreSort().order(order));
+        }
         return sort(SortBuilders.fieldSort(name).order(order));
     }
 
@@ -343,32 +350,27 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
      *            The name of the field to sort by
      */
     public SearchSourceBuilder sort(String name) {
+        if (name.equals(ScoreSortBuilder.NAME)) {
+            return sort(SortBuilders.scoreSort());
+        }
         return sort(SortBuilders.fieldSort(name));
     }
 
     /**
      * Adds a sort builder.
      */
-    public SearchSourceBuilder sort(SortBuilder sort) {
-        try {
+    public SearchSourceBuilder sort(SortBuilder<?> sort) {
             if (sorts == null) {
                 sorts = new ArrayList<>();
             }
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            sort.toXContent(builder, EMPTY_PARAMS);
-            builder.endObject();
-            sorts.add(builder.bytes());
+            sorts.add(sort);
             return this;
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
     }
 
     /**
      * Gets the bytes representing the sort builders for this request.
      */
-    public List<BytesReference> sorts() {
+    public List<SortBuilder<?>> sorts() {
         return sorts;
     }
 
@@ -475,20 +477,14 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     }
 
     public SearchSourceBuilder suggest(SuggestBuilder suggestBuilder) {
-        try {
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            suggestBuilder.toXContent(builder, EMPTY_PARAMS);
-            this.suggestBuilder = builder.bytes();
-            return this;
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
+        this.suggestBuilder = suggestBuilder;
+        return this;
     }
 
     /**
-     * Gets the bytes representing the suggester builder for this request.
+     * Gets the suggester builder for this request.
      */
-    public BytesReference suggest() {
+    public SuggestBuilder suggest() {
         return suggestBuilder;
     }
 
@@ -733,22 +729,86 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         return ext;
     }
 
+    /**
+     * @return true if the source only has suggest
+     */
+    public boolean isSuggestOnly() {
+        return suggestBuilder != null
+            && queryBuilder == null && aggregations == null;
+    }
+
+    /**
+     * Rewrites this search source builder into its primitive form. e.g. by
+     * rewriting the QueryBuilder. If the builder did not change the identity
+     * reference must be returned otherwise the builder will be rewritten
+     * infinitely.
+     */
+    public SearchSourceBuilder rewrite(QueryShardContext context) throws IOException {
+        assert (this.equals(shallowCopy(queryBuilder, postQueryBuilder)));
+        QueryBuilder<?> queryBuilder = null;
+        if (this.queryBuilder != null) {
+            queryBuilder = this.queryBuilder.rewrite(context);
+        }
+        QueryBuilder<?> postQueryBuilder = null;
+        if (this.postQueryBuilder != null) {
+            postQueryBuilder = this.postQueryBuilder.rewrite(context);
+        }
+        boolean rewritten = queryBuilder != this.queryBuilder || postQueryBuilder != this.postQueryBuilder;
+        if (rewritten) {
+            return shallowCopy(queryBuilder, postQueryBuilder);
+        }
+        return this;
+    }
+
+    private SearchSourceBuilder shallowCopy(QueryBuilder<?> queryBuilder, QueryBuilder<?> postQueryBuilder) {
+            SearchSourceBuilder rewrittenBuilder = new SearchSourceBuilder();
+            rewrittenBuilder.aggregations = aggregations;
+            rewrittenBuilder.explain = explain;
+            rewrittenBuilder.ext = ext;
+            rewrittenBuilder.fetchSourceContext = fetchSourceContext;
+            rewrittenBuilder.fieldDataFields = fieldDataFields;
+            rewrittenBuilder.fieldNames = fieldNames;
+            rewrittenBuilder.from = from;
+            rewrittenBuilder.highlightBuilder = highlightBuilder;
+            rewrittenBuilder.indexBoost = indexBoost;
+            rewrittenBuilder.innerHitsBuilder = innerHitsBuilder;
+            rewrittenBuilder.minScore = minScore;
+            rewrittenBuilder.postQueryBuilder = postQueryBuilder;
+            rewrittenBuilder.profile = profile;
+            rewrittenBuilder.queryBuilder = queryBuilder;
+            rewrittenBuilder.rescoreBuilders = rescoreBuilders;
+            rewrittenBuilder.scriptFields = scriptFields;
+            rewrittenBuilder.searchAfterBuilder = searchAfterBuilder;
+            rewrittenBuilder.size = size;
+            rewrittenBuilder.sorts = sorts;
+            rewrittenBuilder.stats = stats;
+            rewrittenBuilder.suggestBuilder = suggestBuilder;
+            rewrittenBuilder.terminateAfter = terminateAfter;
+            rewrittenBuilder.timeoutInMillis = timeoutInMillis;
+            rewrittenBuilder.trackScores = trackScores;
+            rewrittenBuilder.version = version;
+            return rewrittenBuilder;
+        }
+
     /**
      * Create a new SearchSourceBuilder with attributes set by an xContent.
      */
-    public SearchSourceBuilder fromXContent(XContentParser parser, QueryParseContext context, AggregatorParsers aggParsers)
+    public SearchSourceBuilder fromXContent(XContentParser parser, QueryParseContext context,
+                                            AggregatorParsers aggParsers, Suggesters suggesters)
             throws IOException {
         SearchSourceBuilder builder = new SearchSourceBuilder();
-        builder.parseXContent(parser, context, aggParsers);
+        builder.parseXContent(parser, context, aggParsers, suggesters);
         return builder;
     }
 
     /**
      * Parse some xContent into this SearchSourceBuilder, overwriting any values specified in the xContent. Use this if you need to set up
      * different defaults than a regular SearchSourceBuilder would have and use
-     * {@link #fromXContent(XContentParser, QueryParseContext, AggregatorParsers)} if you have normal defaults.
+     * {@link #fromXContent(XContentParser, QueryParseContext, AggregatorParsers, Suggesters)} if you have normal defaults.
      */
-    public void parseXContent(XContentParser parser, QueryParseContext context, AggregatorParsers aggParsers) throws IOException {
+    public void parseXContent(XContentParser parser, QueryParseContext context, AggregatorParsers aggParsers, Suggesters suggesters)
+        throws IOException {
+
         XContentParser.Token token = parser.currentToken();
         String currentFieldName = null;
         if (token != XContentParser.Token.START_OBJECT && (token = parser.nextToken()) != XContentParser.Token.START_OBJECT) {
@@ -852,12 +912,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                     XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser);
                     innerHitsBuilder = xContentBuilder.bytes();
                 } else if (context.parseFieldMatcher().match(currentFieldName, SUGGEST_FIELD)) {
-                    XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser);
-                    suggestBuilder = xContentBuilder.bytes();
+                    suggestBuilder = SuggestBuilder.fromXContent(context, suggesters);
                 } else if (context.parseFieldMatcher().match(currentFieldName, SORT_FIELD)) {
-                    sorts = new ArrayList<>();
-                    XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser);
-                    sorts.add(xContentBuilder.bytes());
+                    sorts = new ArrayList<>(SortBuilder.fromXContent(context));
                 } else if (context.parseFieldMatcher().match(currentFieldName, EXT_FIELD)) {
                     XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser);
                     ext = xContentBuilder.bytes();
@@ -888,11 +945,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                         }
                     }
                 } else if (context.parseFieldMatcher().match(currentFieldName, SORT_FIELD)) {
-                    sorts = new ArrayList<>();
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser);
-                        sorts.add(xContentBuilder.bytes());
-                    }
+                    sorts = new ArrayList<>(SortBuilder.fromXContent(context));
                 } else if (context.parseFieldMatcher().match(currentFieldName, RESCORE_FIELD)) {
                     rescoreBuilders = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
@@ -1005,10 +1058,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
 
         if (sorts != null) {
             builder.startArray(SORT_FIELD.getPreferredName());
-            for (BytesReference sort : sorts) {
-                XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(sort);
-                parser.nextToken();
-                builder.copyCurrentStructure(parser);
+            for (SortBuilder<?> sort : sorts) {
+                sort.toXContent(builder, params);
             }
             builder.endArray();
         }
@@ -1050,10 +1101,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         }
 
         if (suggestBuilder != null) {
-            builder.field(SUGGEST_FIELD.getPreferredName());
-            XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(suggestBuilder);
-            parser.nextToken();
-            builder.copyCurrentStructure(parser);
+            builder.field(SUGGEST_FIELD.getPreferredName(), suggestBuilder);
         }
 
         if (rescoreBuilders != null) {
@@ -1217,9 +1265,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         builder.size = in.readVInt();
         if (in.readBoolean()) {
             int size = in.readVInt();
-            List<BytesReference> sorts = new ArrayList<>();
+            List<SortBuilder<?>> sorts = new ArrayList<>();
             for (int i = 0; i < size; i++) {
-                sorts.add(in.readBytesReference());
+                sorts.add(in.readSortBuilder());
             }
             builder.sorts = sorts;
         }
@@ -1232,7 +1280,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
             builder.stats = stats;
         }
         if (in.readBoolean()) {
-            builder.suggestBuilder = in.readBytesReference();
+            builder.suggestBuilder = SuggestBuilder.PROTOTYPE.readFrom(in);
         }
         builder.terminateAfter = in.readVInt();
         builder.timeoutInMillis = in.readLong();
@@ -1333,8 +1381,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         out.writeBoolean(hasSorts);
         if (hasSorts) {
             out.writeVInt(sorts.size());
-            for (BytesReference sort : sorts) {
-                out.writeBytesReference(sort);
+            for (SortBuilder<?> sort : sorts) {
+                out.writeSortBuilder(sort);
             }
         }
         boolean hasStats = stats != null;
@@ -1348,7 +1396,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         boolean hasSuggestBuilder = suggestBuilder != null;
         out.writeBoolean(hasSuggestBuilder);
         if (hasSuggestBuilder) {
-            out.writeBytesReference(suggestBuilder);
+            suggestBuilder.writeTo(out);
         }
         out.writeVInt(terminateAfter);
         out.writeLong(timeoutInMillis);
diff --git a/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java b/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
index ad8e71f5b93..d5d4607fba9 100644
--- a/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
+++ b/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
@@ -76,7 +76,7 @@ public class SearchPhaseController extends AbstractComponent {
         public int compare(AtomicArray.Entry<? extends QuerySearchResultProvider> o1, AtomicArray.Entry<? extends QuerySearchResultProvider> o2) {
             int i = o1.value.shardTarget().index().compareTo(o2.value.shardTarget().index());
             if (i == 0) {
-                i = o1.value.shardTarget().shardId() - o2.value.shardTarget().shardId();
+                i = o1.value.shardTarget().shardId().id() - o2.value.shardTarget().shardId().id();
             }
             return i;
         }
@@ -386,7 +386,7 @@ public class SearchPhaseController extends AbstractComponent {
                 Suggest.group(groupedSuggestions, shardResult);
             }
 
-            suggest = hasSuggestions ? new Suggest(Suggest.Fields.SUGGEST, Suggest.reduce(groupedSuggestions)) : null;
+            suggest = hasSuggestions ? new Suggest(Suggest.reduce(groupedSuggestions)) : null;
         }
 
         // merge addAggregation
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/ScrollQueryFetchSearchResult.java b/core/src/main/java/org/elasticsearch/search/fetch/ScrollQueryFetchSearchResult.java
index fb0fc75299f..dbaee5b64bb 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/ScrollQueryFetchSearchResult.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/ScrollQueryFetchSearchResult.java
@@ -26,7 +26,6 @@ import org.elasticsearch.transport.TransportResponse;
 
 import java.io.IOException;
 
-import static org.elasticsearch.search.SearchShardTarget.readSearchShardTarget;
 import static org.elasticsearch.search.fetch.QueryFetchSearchResult.readQueryFetchSearchResult;
 
 /**
@@ -56,7 +55,7 @@ public class ScrollQueryFetchSearchResult extends TransportResponse {
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        shardTarget = readSearchShardTarget(in);
+        shardTarget = new SearchShardTarget(in);
         result = readQueryFetchSearchResult(in);
         result.shardTarget(shardTarget);
     }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
index 39c432f9cc9..f9cf3f09f39 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
@@ -35,7 +35,6 @@ import org.elasticsearch.search.highlight.HighlighterParseElement;
 import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.InternalSearchHits;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.sort.SortParseElement;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -51,8 +50,8 @@ public class InnerHitsFetchSubPhase implements FetchSubPhase {
     private FetchPhase fetchPhase;
 
     @Inject
-    public InnerHitsFetchSubPhase(SortParseElement sortParseElement, FetchSourceParseElement sourceParseElement, HighlighterParseElement highlighterParseElement, FieldDataFieldsParseElement fieldDataFieldsParseElement, ScriptFieldsParseElement scriptFieldsParseElement) {
-        parseElements = singletonMap("inner_hits", new InnerHitsParseElement(sortParseElement, sourceParseElement, highlighterParseElement,
+    public InnerHitsFetchSubPhase(FetchSourceParseElement sourceParseElement, HighlighterParseElement highlighterParseElement, FieldDataFieldsParseElement fieldDataFieldsParseElement, ScriptFieldsParseElement scriptFieldsParseElement) {
+        parseElements = singletonMap("inner_hits", new InnerHitsParseElement(sourceParseElement, highlighterParseElement,
                 fieldDataFieldsParseElement, scriptFieldsParseElement));
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java
index 077268ac52f..fc1b2bf399f 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java
@@ -32,7 +32,6 @@ import org.elasticsearch.search.fetch.source.FetchSourceParseElement;
 import org.elasticsearch.search.highlight.HighlighterParseElement;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SubSearchContext;
-import org.elasticsearch.search.sort.SortParseElement;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -43,14 +42,12 @@ import static org.elasticsearch.index.query.support.InnerHitsQueryParserHelper.p
  */
 public class InnerHitsParseElement implements SearchParseElement {
 
-    private final SortParseElement sortParseElement;
     private final FetchSourceParseElement sourceParseElement;
     private final HighlighterParseElement highlighterParseElement;
     private final FieldDataFieldsParseElement fieldDataFieldsParseElement;
     private final ScriptFieldsParseElement scriptFieldsParseElement;
 
-    public InnerHitsParseElement(SortParseElement sortParseElement, FetchSourceParseElement sourceParseElement, HighlighterParseElement highlighterParseElement, FieldDataFieldsParseElement fieldDataFieldsParseElement, ScriptFieldsParseElement scriptFieldsParseElement) {
-        this.sortParseElement = sortParseElement;
+    public InnerHitsParseElement(FetchSourceParseElement sourceParseElement, HighlighterParseElement highlighterParseElement, FieldDataFieldsParseElement fieldDataFieldsParseElement, ScriptFieldsParseElement scriptFieldsParseElement) {
         this.sourceParseElement = sourceParseElement;
         this.highlighterParseElement = highlighterParseElement;
         this.fieldDataFieldsParseElement = fieldDataFieldsParseElement;
@@ -184,10 +181,10 @@ public class InnerHitsParseElement implements SearchParseElement {
                 } else if ("inner_hits".equals(fieldName)) {
                     childInnerHits = parseInnerHits(parser, context, searchContext);
                 } else {
-                    parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sortParseElement, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement);
+                    parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement);
                 }
             } else {
-                parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sortParseElement, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement);
+                parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement);
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
index 8c3c19343b4..f3fe48f6682 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
@@ -49,6 +49,7 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
+import org.elasticsearch.index.percolator.PercolatorQueryCache;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryShardContext;
@@ -190,6 +191,9 @@ public class DefaultSearchContext extends SearchContext {
      */
     @Override
     public void preProcess() {
+        if (hasOnlySuggest() ) {
+            return;
+        }
         if (scrollContext == null) {
             long from = from() == -1 ? 0 : from();
             long size = size() == -1 ? 10 : size();
@@ -487,6 +491,11 @@ public class DefaultSearchContext extends SearchContext {
         return indexService.fieldData();
     }
 
+    @Override
+    public PercolatorQueryCache percolatorQueryCache() {
+        return indexService.cache().getPercolatorQueryCache();
+    }
+
     @Override
     public long timeoutInMillis() {
         return timeoutInMillis;
diff --git a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
index 83ea2b1ccd8..fedab3f9782 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
@@ -34,6 +34,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
+import org.elasticsearch.index.percolator.PercolatorQueryCache;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexShard;
@@ -274,6 +275,11 @@ public abstract class FilteredSearchContext extends SearchContext {
         return in.fieldData();
     }
 
+    @Override
+    public PercolatorQueryCache percolatorQueryCache() {
+        return in.percolatorQueryCache();
+    }
+
     @Override
     public long timeoutInMillis() {
         return in.timeoutInMillis();
diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
index c6afe325bb3..dcbcce503a4 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
@@ -55,7 +55,6 @@ import static java.util.Collections.singletonMap;
 import static java.util.Collections.unmodifiableMap;
 import static org.elasticsearch.common.lucene.Lucene.readExplanation;
 import static org.elasticsearch.common.lucene.Lucene.writeExplanation;
-import static org.elasticsearch.search.SearchShardTarget.readSearchShardTarget;
 import static org.elasticsearch.search.highlight.HighlightField.readHighlightField;
 import static org.elasticsearch.search.internal.InternalSearchHitField.readSearchHitField;
 
@@ -638,7 +637,7 @@ public class InternalSearchHit implements SearchHit {
 
         if (context.streamShardTarget() == ShardTargetType.STREAM) {
             if (in.readBoolean()) {
-                shard = readSearchShardTarget(in);
+                shard = new SearchShardTarget(in);
             }
         } else if (context.streamShardTarget() == ShardTargetType.LOOKUP) {
             int lookupId = in.readVInt();
diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHits.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHits.java
index 9e787cf2aa9..09d11e1a1a3 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHits.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHits.java
@@ -34,7 +34,6 @@ import java.util.IdentityHashMap;
 import java.util.Iterator;
 import java.util.Map;
 
-import static org.elasticsearch.search.SearchShardTarget.readSearchShardTarget;
 import static org.elasticsearch.search.internal.InternalSearchHit.readSearchHit;
 
 /**
@@ -216,7 +215,7 @@ public class InternalSearchHits implements SearchHits {
                 // read the lookup table first
                 int lookupSize = in.readVInt();
                 for (int i = 0; i < lookupSize; i++) {
-                    context.handleShardLookup().put(in.readVInt(), readSearchShardTarget(in));
+                    context.handleShardLookup().put(in.readVInt(), new SearchShardTarget(in));
                 }
             }
 
@@ -262,4 +261,4 @@ public class InternalSearchHits implements SearchHits {
             }
         }
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java
index b8255e0bb52..1a2e1f70191 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java
@@ -134,7 +134,7 @@ public class InternalSearchResponse implements Streamable, ToXContent {
             aggregations = InternalAggregations.readAggregations(in);
         }
         if (in.readBoolean()) {
-            suggest = Suggest.readSuggest(Suggest.Fields.SUGGEST, in);
+            suggest = Suggest.readSuggest(in);
         }
         timedOut = in.readBoolean();
 
diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index 2b35e182161..f9d0b6b9283 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -38,6 +38,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
+import org.elasticsearch.index.percolator.PercolatorQueryCache;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexShard;
@@ -148,6 +149,10 @@ public abstract class SearchContext implements Releasable {
         return nowInMillisUsed;
     }
 
+    public final void resetNowInMillisUsed() {
+        this.nowInMillisUsed = false;
+    }
+
     protected abstract long nowInMillisImpl();
 
     public abstract ScrollContext scrollContext();
@@ -217,6 +222,8 @@ public abstract class SearchContext implements Releasable {
 
     public abstract IndexFieldDataService fieldData();
 
+    public abstract PercolatorQueryCache percolatorQueryCache();
+
     public abstract long timeoutInMillis();
 
     public abstract void timeoutInMillis(long timeoutInMillis);
@@ -346,6 +353,14 @@ public abstract class SearchContext implements Releasable {
         }
     }
 
+    /**
+     * @return true if the request contains only suggest
+     */
+    public final boolean hasOnlySuggest() {
+        return request().source() != null
+            && request().source().isSuggestOnly();
+    }
+
     /**
      * Looks up the given field, but does not restrict to fields in the types set on this context.
      */
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java
index 0f46461f4a2..31192350308 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.script.Template;
 import org.elasticsearch.search.Scroll;
@@ -58,8 +59,7 @@ import static org.elasticsearch.search.Scroll.readScroll;
 
 public class ShardSearchLocalRequest implements ShardSearchRequest {
 
-    private String index;
-    private int shardId;
+    private ShardId shardId;
     private int numberOfShards;
     private SearchType searchType;
     private Scroll scroll;
@@ -97,8 +97,7 @@ public class ShardSearchLocalRequest implements ShardSearchRequest {
 
     public ShardSearchLocalRequest(ShardId shardId, int numberOfShards, SearchType searchType, SearchSourceBuilder source, String[] types,
             Boolean requestCache) {
-        this.index = shardId.getIndexName();
-        this.shardId = shardId.id();
+        this.shardId = shardId;
         this.numberOfShards = numberOfShards;
         this.searchType = searchType;
         this.source = source;
@@ -106,13 +105,9 @@ public class ShardSearchLocalRequest implements ShardSearchRequest {
         this.requestCache = requestCache;
     }
 
-    @Override
-    public String index() {
-        return index;
-    }
 
     @Override
-    public int shardId() {
+    public ShardId shardId() {
         return shardId;
     }
 
@@ -177,8 +172,7 @@ public class ShardSearchLocalRequest implements ShardSearchRequest {
 
     @SuppressWarnings("unchecked")
     protected void innerReadFrom(StreamInput in) throws IOException {
-        index = in.readString();
-        shardId = in.readVInt();
+        shardId = ShardId.readShardId(in);
         searchType = SearchType.fromId(in.readByte());
         numberOfShards = in.readVInt();
         if (in.readBoolean()) {
@@ -195,8 +189,7 @@ public class ShardSearchLocalRequest implements ShardSearchRequest {
     }
 
     protected void innerWriteTo(StreamOutput out, boolean asKey) throws IOException {
-        out.writeString(index);
-        out.writeVInt(shardId);
+        shardId.writeTo(out);
         out.writeByte(searchType.id());
         if (!asKey) {
             out.writeVInt(numberOfShards);
@@ -232,4 +225,15 @@ public class ShardSearchLocalRequest implements ShardSearchRequest {
         // we could potentially keep it without copying, but then pay the price of extra unused bytes up to a page
         return out.bytes().copyBytesArray();
     }
+
+    @Override
+    public void rewrite(QueryShardContext context) throws IOException {
+        SearchSourceBuilder source = this.source;
+        SearchSourceBuilder rewritten = null;
+        while (rewritten != source) {
+            rewritten = source.rewrite(context);
+            source = rewritten;
+        }
+        this.source = source;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
index 1f0b3d1f188..aa148e215c8 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
@@ -21,6 +21,8 @@ package org.elasticsearch.search.internal;
 
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.script.Template;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -34,9 +36,7 @@ import java.io.IOException;
  */
 public interface ShardSearchRequest {
 
-    String index();
-
-    int shardId();
+    ShardId shardId();
 
     String[] types();
 
@@ -73,4 +73,10 @@ public interface ShardSearchRequest {
      * Returns the cache key for this shard search request, based on its content
      */
     BytesReference cacheKey() throws IOException;
+
+    /**
+     * Rewrites this request into its primitive form. e.g. by rewriting the
+     * QueryBuilder.
+     */
+    void rewrite(QueryShardContext context) throws IOException;
 }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java
index 48ea31c170a..cd6460a686f 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java
@@ -28,6 +28,8 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.script.Template;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -71,13 +73,9 @@ public class ShardSearchTransportRequest extends TransportRequest implements Sha
         return originalIndices.indicesOptions();
     }
 
-    @Override
-    public String index() {
-        return shardSearchLocalRequest.index();
-    }
 
     @Override
-    public int shardId() {
+    public ShardId shardId() {
         return shardSearchLocalRequest.shardId();
     }
 
@@ -159,4 +157,16 @@ public class ShardSearchTransportRequest extends TransportRequest implements Sha
     public boolean isProfile() {
         return shardSearchLocalRequest.isProfile();
     }
+
+    @Override
+    public void rewrite(QueryShardContext context) throws IOException {
+        shardSearchLocalRequest.rewrite(context);
+    }
+
+    private ShardSearchTransportRequest shallowCopy(ShardSearchLocalRequest rewritten) {
+        ShardSearchTransportRequest newRequest = new ShardSearchTransportRequest();
+        newRequest.originalIndices = originalIndices;
+        newRequest.shardSearchLocalRequest = rewritten;
+        return newRequest;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index 5a98744505a..62210655a00 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -58,7 +58,6 @@ import org.elasticsearch.search.profile.ProfileShardResult;
 import org.elasticsearch.search.profile.Profiler;
 import org.elasticsearch.search.rescore.RescorePhase;
 import org.elasticsearch.search.rescore.RescoreSearchContext;
-import org.elasticsearch.search.sort.SortParseElement;
 import org.elasticsearch.search.sort.TrackScoresParseElement;
 import org.elasticsearch.search.suggest.SuggestPhase;
 
@@ -98,7 +97,6 @@ public class QueryPhase implements SearchPhase {
         parseElements.put("query", new QueryParseElement());
         parseElements.put("post_filter", new PostFilterParseElement());
         parseElements.put("postFilter", new PostFilterParseElement());
-        parseElements.put("sort", new SortParseElement());
         parseElements.put("trackScores", new TrackScoresParseElement());
         parseElements.put("track_scores", new TrackScoresParseElement());
         parseElements.put("min_score", new MinScoreParseElement());
@@ -118,6 +116,12 @@ public class QueryPhase implements SearchPhase {
 
     @Override
     public void execute(SearchContext searchContext) throws QueryPhaseExecutionException {
+        if (searchContext.hasOnlySuggest()) {
+            suggestPhase.execute(searchContext);
+            // TODO: fix this once we can fetch docs for suggestions
+            searchContext.queryResult().topDocs(new TopDocs(0, Lucene.EMPTY_SCORE_DOCS, 0));
+            return;
+        }
         // Pre-process aggregations as late as possible. In the case of a DFS_Q_T_F
         // request, preProcess is called on the DFS phase phase, this is why we pre-process them
         // here to make sure it happens during the QUERY phase
diff --git a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
index 9223eb5a82d..2b82633ebfd 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
@@ -207,7 +207,7 @@ public class QuerySearchResult extends QuerySearchResultProvider {
             this.pipelineAggregators = pipelineAggregators;
         }
         if (in.readBoolean()) {
-            suggest = Suggest.readSuggest(Suggest.Fields.SUGGEST, in);
+            suggest = Suggest.readSuggest(in);
         }
         searchTimedOut = in.readBoolean();
         terminatedEarly = in.readOptionalBoolean();
diff --git a/core/src/main/java/org/elasticsearch/search/query/ScrollQuerySearchResult.java b/core/src/main/java/org/elasticsearch/search/query/ScrollQuerySearchResult.java
index ebb7615da44..bcdd94adf89 100644
--- a/core/src/main/java/org/elasticsearch/search/query/ScrollQuerySearchResult.java
+++ b/core/src/main/java/org/elasticsearch/search/query/ScrollQuerySearchResult.java
@@ -26,7 +26,6 @@ import org.elasticsearch.transport.TransportResponse;
 
 import java.io.IOException;
 
-import static org.elasticsearch.search.SearchShardTarget.readSearchShardTarget;
 import static org.elasticsearch.search.query.QuerySearchResult.readQuerySearchResult;
 
 /**
@@ -56,7 +55,7 @@ public class ScrollQuerySearchResult extends TransportResponse {
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        shardTarget = readSearchShardTarget(in);
+        shardTarget = new SearchShardTarget(in);
         queryResult = readQuerySearchResult(in);
         queryResult.shardTarget(shardTarget);
     }
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescoreMode.java b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescoreMode.java
index b0d5a325e5a..959bd51270b 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescoreMode.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescoreMode.java
@@ -85,10 +85,7 @@ public enum QueryRescoreMode implements Writeable<QueryRescoreMode> {
 
     public abstract float combine(float primary, float secondary);
 
-    static QueryRescoreMode PROTOTYPE = Total;
-
-    @Override
-    public QueryRescoreMode readFrom(StreamInput in) throws IOException {
+    public static QueryRescoreMode readFromStream(StreamInput in) throws IOException {
         int ordinal = in.readVInt();
         if (ordinal < 0 || ordinal >= values().length) {
             throw new IOException("Unknown ScoreMode ordinal [" + ordinal + "]");
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java
index c65fca79a97..85564265579 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java
@@ -25,7 +25,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
@@ -39,8 +38,6 @@ public class QueryRescorerBuilder extends RescoreBuilder<QueryRescorerBuilder> {
 
     public static final String NAME = "query";
 
-    public static final QueryRescorerBuilder PROTOTYPE = new QueryRescorerBuilder(new MatchAllQueryBuilder());
-
     public static final float DEFAULT_RESCORE_QUERYWEIGHT = 1.0f;
     public static final float DEFAULT_QUERYWEIGHT = 1.0f;
     public static final QueryRescoreMode DEFAULT_SCORE_MODE = QueryRescoreMode.Total;
@@ -77,6 +74,25 @@ public class QueryRescorerBuilder extends RescoreBuilder<QueryRescorerBuilder> {
         this.queryBuilder = builder;
     }
 
+    /**
+     * Read from a stream.
+     */
+    public QueryRescorerBuilder(StreamInput in) throws IOException {
+        super(in);
+        queryBuilder = in.readQuery();
+        scoreMode = QueryRescoreMode.readFromStream(in);
+        rescoreQueryWeight = in.readFloat();
+        queryWeight = in.readFloat();
+    }
+
+    @Override
+    public void doWriteTo(StreamOutput out) throws IOException {
+        out.writeQuery(queryBuilder);
+        scoreMode.writeTo(out);
+        out.writeFloat(rescoreQueryWeight);
+        out.writeFloat(queryWeight);
+    }
+
     /**
      * @return the query used for this rescore query
      */
@@ -140,9 +156,9 @@ public class QueryRescorerBuilder extends RescoreBuilder<QueryRescorerBuilder> {
         builder.endObject();
     }
 
-    public QueryRescorerBuilder fromXContent(QueryParseContext parseContext) throws IOException {
-            InnerBuilder innerBuilder = QUERY_RESCORE_PARSER.parse(parseContext.parser(), new InnerBuilder(), parseContext);
-            return innerBuilder.build();
+    public static QueryRescorerBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+        InnerBuilder innerBuilder = QUERY_RESCORE_PARSER.parse(parseContext.parser(), new InnerBuilder(), parseContext);
+        return innerBuilder.build();
     }
 
     @Override
@@ -181,23 +197,6 @@ public class QueryRescorerBuilder extends RescoreBuilder<QueryRescorerBuilder> {
                Objects.equals(queryBuilder, other.queryBuilder);
     }
 
-    @Override
-    public QueryRescorerBuilder doReadFrom(StreamInput in) throws IOException {
-        QueryRescorerBuilder rescorer = new QueryRescorerBuilder(in.readQuery());
-        rescorer.setScoreMode(QueryRescoreMode.PROTOTYPE.readFrom(in));
-        rescorer.setRescoreQueryWeight(in.readFloat());
-        rescorer.setQueryWeight(in.readFloat());
-        return rescorer;
-    }
-
-    @Override
-    public void doWriteTo(StreamOutput out) throws IOException {
-        out.writeQuery(queryBuilder);
-        scoreMode.writeTo(out);
-        out.writeFloat(rescoreQueryWeight);
-        out.writeFloat(queryWeight);
-    }
-
     @Override
     public String getWriteableName() {
         return NAME;
@@ -208,7 +207,7 @@ public class QueryRescorerBuilder extends RescoreBuilder<QueryRescorerBuilder> {
      * for the constructor of {@link QueryRescorerBuilder}, but {@link ObjectParser} only
      * allows filling properties of an already constructed value.
      */
-    private class InnerBuilder {
+    private static class InnerBuilder {
 
         private QueryBuilder<?> queryBuilder;
         private float rescoreQueryWeight = DEFAULT_RESCORE_QUERYWEIGHT;
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java b/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java
index 8dad07a5430..32885380866 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java
@@ -46,6 +46,27 @@ public abstract class RescoreBuilder<RB extends RescoreBuilder<RB>> implements T
 
     private static ParseField WINDOW_SIZE_FIELD = new ParseField("window_size");
 
+    /**
+     * Construct an empty RescoreBuilder.
+     */
+    public RescoreBuilder() {
+    }
+
+    /**
+     * Read from a stream.
+     */
+    protected RescoreBuilder(StreamInput in) throws IOException {
+        windowSize = in.readOptionalVInt();
+    }
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalVInt(this.windowSize);
+        doWriteTo(out);
+    }
+
+    protected abstract void doWriteTo(StreamOutput out) throws IOException;
+
     @SuppressWarnings("unchecked")
     public RB windowSize(int windowSize) {
         this.windowSize = windowSize;
@@ -74,7 +95,7 @@ public abstract class RescoreBuilder<RB extends RescoreBuilder<RB>> implements T
             } else if (token == XContentParser.Token.START_OBJECT) {
                 // we only have QueryRescorer at this point
                 if (QueryRescorerBuilder.NAME.equals(fieldName)) {
-                    rescorer = QueryRescorerBuilder.PROTOTYPE.fromXContent(parseContext);
+                    rescorer = QueryRescorerBuilder.fromXContent(parseContext);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "rescore doesn't support rescorer with name [" + fieldName + "]");
                 }
@@ -128,23 +149,6 @@ public abstract class RescoreBuilder<RB extends RescoreBuilder<RB>> implements T
         return Objects.equals(windowSize, other.windowSize);
     }
 
-    @Override
-    public RB readFrom(StreamInput in) throws IOException {
-        RB builder = doReadFrom(in);
-        builder.windowSize = in.readOptionalVInt();
-        return builder;
-    }
-
-    protected abstract RB doReadFrom(StreamInput in) throws IOException;
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        doWriteTo(out);
-        out.writeOptionalVInt(this.windowSize);
-    }
-
-    protected abstract void doWriteTo(StreamOutput out) throws IOException;
-
     @Override
     public final String toString() {
         try {
diff --git a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
index 4f082b057da..1798c75c6a2 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
@@ -19,34 +19,75 @@
 
 package org.elasticsearch.search.sort;
 
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
+import java.util.Objects;
 
 /**
  * A sort builder to sort based on a document field.
  */
-public class FieldSortBuilder extends SortBuilder {
+public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
+    public static final String NAME = "field_sort";
+    public static final ParseField NESTED_PATH = new ParseField("nested_path");
+    public static final ParseField NESTED_FILTER = new ParseField("nested_filter");
+    public static final ParseField MISSING = new ParseField("missing");
+    public static final ParseField ORDER = new ParseField("order");
+    public static final ParseField REVERSE = new ParseField("reverse");
+    public static final ParseField SORT_MODE = new ParseField("mode");
+    public static final ParseField UNMAPPED_TYPE = new ParseField("unmapped_type");
+
+    /**
+     * special field name to sort by index order
+     */
+    public static final String DOC_FIELD_NAME = "_doc";
+    private static final SortField SORT_DOC = new SortField(null, SortField.Type.DOC);
+    private static final SortField SORT_DOC_REVERSE = new SortField(null, SortField.Type.DOC, true);
 
     private final String fieldName;
 
-    private SortOrder order;
-
     private Object missing;
 
     private String unmappedType;
 
-    private String sortMode;
+    private SortMode sortMode;
 
-    private QueryBuilder nestedFilter;
+    private QueryBuilder<?> nestedFilter;
 
     private String nestedPath;
 
+    /** Copy constructor. */
+    public FieldSortBuilder(FieldSortBuilder template) {
+        this(template.fieldName);
+        this.order(template.order());
+        this.missing(template.missing());
+        this.unmappedType(template.unmappedType());
+        if (template.sortMode != null) {
+            this.sortMode(template.sortMode());
+        }
+        this.setNestedFilter(template.getNestedFilter());
+        this.setNestedPath(template.getNestedPath());
+    }
+
     /**
      * Constructs a new sort based on a document field.
      *
-     * @param fieldName The field name.
+     * @param fieldName
+     *            The field name.
      */
     public FieldSortBuilder(String fieldName) {
         if (fieldName == null) {
@@ -56,30 +97,55 @@ public class FieldSortBuilder extends SortBuilder {
     }
 
     /**
-     * The order of sorting. Defaults to {@link SortOrder#ASC}.
+     * Read from a stream.
      */
+    public FieldSortBuilder(StreamInput in) throws IOException {
+        fieldName = in.readString();
+        nestedFilter = in.readOptionalQuery();
+        nestedPath = in.readOptionalString();
+        missing = in.readGenericValue();
+        order = in.readOptionalWriteable(SortOrder::readFromStream);
+        sortMode = in.readOptionalWriteable(SortMode::readFromStream);
+        unmappedType = in.readOptionalString();
+    }
+
     @Override
-    public FieldSortBuilder order(SortOrder order) {
-        this.order = order;
-        return this;
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(fieldName);
+        out.writeOptionalQuery(nestedFilter);
+        out.writeOptionalString(nestedPath);
+        out.writeGenericValue(missing);
+        out.writeOptionalWriteable(order);
+        out.writeOptionalWriteable(sortMode);
+        out.writeOptionalString(unmappedType);
+    }
+
+    /** Returns the document field this sort should be based on. */
+    public String getFieldName() {
+        return this.fieldName;
     }
 
     /**
      * Sets the value when a field is missing in a doc. Can also be set to <tt>_last</tt> or
      * <tt>_first</tt> to sort missing last or first respectively.
      */
-    @Override
     public FieldSortBuilder missing(Object missing) {
         this.missing = missing;
         return this;
     }
 
+    /** Returns the value used when a field is missing in a doc. */
+    public Object missing() {
+        return missing;
+    }
+
     /**
      * Set the type to use in case the current field is not mapped in an index.
-     * Specifying a type tells Elasticsearch what type the sort values should have, which is important
-     * for cross-index search, if there are sort fields that exist on some indices only.
-     * If the unmapped type is <tt>null</tt> then query execution will fail if one or more indices
-     * don't have a mapping for the current field.
+     * Specifying a type tells Elasticsearch what type the sort values should
+     * have, which is important for cross-index search, if there are sort fields
+     * that exist on some indices only. If the unmapped type is <tt>null</tt>
+     * then query execution will fail if one or more indices don't have a
+     * mapping for the current field.
      */
     public FieldSortBuilder unmappedType(String type) {
         this.unmappedType = type;
@@ -87,57 +153,246 @@ public class FieldSortBuilder extends SortBuilder {
     }
 
     /**
-     * Defines what values to pick in the case a document contains multiple values for the targeted sort field.
-     * Possible values: min, max, sum and avg
+     * Returns the type to use in case the current field is not mapped in an
+     * index.
+     */
+    public String unmappedType() {
+        return this.unmappedType;
+    }
+
+    /**
+     * Defines what values to pick in the case a document contains multiple
+     * values for the targeted sort field. Possible values: min, max, sum and
+     * avg
+     *
      * <p>
      * The last two values are only applicable for number based fields.
      */
-    public FieldSortBuilder sortMode(String sortMode) {
+    public FieldSortBuilder sortMode(SortMode sortMode) {
+        Objects.requireNonNull(sortMode, "sort mode cannot be null");
         this.sortMode = sortMode;
         return this;
     }
 
     /**
-     * Sets the nested filter that the nested objects should match with in order to be taken into account
-     * for sorting.
+     * Returns what values to pick in the case a document contains multiple
+     * values for the targeted sort field.
      */
-    public FieldSortBuilder setNestedFilter(QueryBuilder nestedFilter) {
+    public SortMode sortMode() {
+        return this.sortMode;
+    }
+
+    /**
+     * Sets the nested filter that the nested objects should match with in order
+     * to be taken into account for sorting.
+     *
+     * TODO should the above getters and setters be deprecated/ changed in
+     * favour of real getters and setters?
+     */
+    public FieldSortBuilder setNestedFilter(QueryBuilder<?> nestedFilter) {
         this.nestedFilter = nestedFilter;
         return this;
     }
 
+    /**
+     * Returns the nested filter that the nested objects should match with in
+     * order to be taken into account for sorting.
+     */
+    public QueryBuilder<?> getNestedFilter() {
+        return this.nestedFilter;
+    }
 
     /**
-     * Sets the nested path if sorting occurs on a field that is inside a nested object. By default when sorting on a
-     * field inside a nested object, the nearest upper nested object is selected as nested path.
+     * Sets the nested path if sorting occurs on a field that is inside a nested
+     * object. By default when sorting on a field inside a nested object, the
+     * nearest upper nested object is selected as nested path.
      */
     public FieldSortBuilder setNestedPath(String nestedPath) {
         this.nestedPath = nestedPath;
         return this;
     }
 
+    /**
+     * Returns the nested path if sorting occurs in a field that is inside a
+     * nested object.
+     */
+    public String getNestedPath() {
+        return this.nestedPath;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startObject(fieldName);
-        if (order != null) {
-            builder.field("order", order.toString());
-        }
+        builder.field(ORDER_FIELD.getPreferredName(), order);
         if (missing != null) {
-            builder.field("missing", missing);
+            builder.field(MISSING.getPreferredName(), missing);
         }
         if (unmappedType != null) {
-            builder.field(SortParseElement.UNMAPPED_TYPE.getPreferredName(), unmappedType);
+            builder.field(UNMAPPED_TYPE.getPreferredName(), unmappedType);
         }
         if (sortMode != null) {
-            builder.field("mode", sortMode);
+            builder.field(SORT_MODE.getPreferredName(), sortMode);
         }
         if (nestedFilter != null) {
-            builder.field("nested_filter", nestedFilter, params);
+            builder.field(NESTED_FILTER.getPreferredName(), nestedFilter, params);
         }
         if (nestedPath != null) {
-            builder.field("nested_path", nestedPath);
+            builder.field(NESTED_PATH.getPreferredName(), nestedPath);
         }
         builder.endObject();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public SortField build(QueryShardContext context) throws IOException {
+        if (DOC_FIELD_NAME.equals(fieldName)) {
+            if (order == SortOrder.DESC) {
+                return SORT_DOC_REVERSE;
+            } else {
+                return SORT_DOC;
+            }
+        } else {
+            MappedFieldType fieldType = context.fieldMapper(fieldName);
+            if (fieldType == null) {
+                if (unmappedType != null) {
+                    fieldType = context.getMapperService().unmappedFieldType(unmappedType);
+                } else {
+                    throw new QueryShardException(context, "No mapping found for [" + fieldName + "] in order to sort on");
+                }
+            }
+
+            if (!fieldType.isSortable()) {
+                throw new QueryShardException(context, "Sorting not supported for field[" + fieldName + "]");
+            }
+
+            MultiValueMode localSortMode = null;
+            if (sortMode != null) {
+                localSortMode = MultiValueMode.fromString(sortMode.toString());
+            }
+
+            if (fieldType.isNumeric() == false && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) {
+                throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields");
+            }
+
+            boolean reverse = (order == SortOrder.DESC);
+            if (localSortMode == null) {
+                localSortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN;
+            }
+
+            final Nested nested = resolveNested(context, nestedPath, nestedFilter);
+            IndexFieldData.XFieldComparatorSource fieldComparatorSource = context.getForField(fieldType)
+                    .comparatorSource(missing, localSortMode, nested);
+            return new SortField(fieldType.name(), fieldComparatorSource, reverse);
+        }
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        FieldSortBuilder builder = (FieldSortBuilder) other;
+        return (Objects.equals(this.fieldName, builder.fieldName) && Objects.equals(this.nestedFilter, builder.nestedFilter)
+                && Objects.equals(this.nestedPath, builder.nestedPath) && Objects.equals(this.missing, builder.missing)
+                && Objects.equals(this.order, builder.order) && Objects.equals(this.sortMode, builder.sortMode)
+                && Objects.equals(this.unmappedType, builder.unmappedType));
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(this.fieldName, this.nestedFilter, this.nestedPath, this.missing, this.order, this.sortMode, this.unmappedType);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    /**
+     * Creates a new {@link FieldSortBuilder} from the query held by the {@link QueryParseContext} in
+     * {@link org.elasticsearch.common.xcontent.XContent} format.
+     *
+     * @param context the input parse context. The state on the parser contained in this context will be changed as a side effect of this
+     *        method call
+     * @param fieldName in some sort syntax variations the field name precedes the xContent object that specifies further parameters, e.g.
+     *        in '{ "foo": { "order" : "asc"} }'. When parsing the inner object, the field name can be passed in via this argument
+     */
+    public static FieldSortBuilder fromXContent(QueryParseContext context, String fieldName) throws IOException {
+        XContentParser parser = context.parser();
+
+        QueryBuilder<?> nestedFilter = null;
+        String nestedPath = null;
+        Object missing = null;
+        SortOrder order = null;
+        SortMode sortMode = null;
+        String unmappedType = null;
+
+        String currentFieldName = null;
+        XContentParser.Token token;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (context.parseFieldMatcher().match(currentFieldName, NESTED_FILTER)) {
+                    nestedFilter = context.parseInnerQueryBuilder();
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "Expected " + NESTED_FILTER.getPreferredName() + " element.");
+                }
+            } else if (token.isValue()) {
+                if (context.parseFieldMatcher().match(currentFieldName, NESTED_PATH)) {
+                    nestedPath = parser.text();
+                } else if (context.parseFieldMatcher().match(currentFieldName, MISSING)) {
+                    missing = parser.objectText();
+                } else if (context.parseFieldMatcher().match(currentFieldName, REVERSE)) {
+                    if (parser.booleanValue()) {
+                        order = SortOrder.DESC;
+                    }
+                    // else we keep the default ASC
+                } else if (context.parseFieldMatcher().match(currentFieldName, ORDER)) {
+                    String sortOrder = parser.text();
+                    if ("asc".equals(sortOrder)) {
+                        order = SortOrder.ASC;
+                    } else if ("desc".equals(sortOrder)) {
+                        order = SortOrder.DESC;
+                    } else {
+                        throw new IllegalStateException("Sort order " + sortOrder + " not supported.");
+                    }
+                } else if (context.parseFieldMatcher().match(currentFieldName, SORT_MODE)) {
+                    sortMode = SortMode.fromString(parser.text());
+                } else if (context.parseFieldMatcher().match(currentFieldName, UNMAPPED_TYPE)) {
+                    unmappedType = parser.text();
+                } else {
+                    throw new IllegalArgumentException("Option " + currentFieldName + " not supported.");
+                }
+            }
+        }
+
+        FieldSortBuilder builder = new FieldSortBuilder(fieldName);
+        if (nestedFilter != null) {
+            builder.setNestedFilter(nestedFilter);
+        }
+        if (nestedPath != null) {
+            builder.setNestedPath(nestedPath);
+        }
+        if (missing != null) {
+            builder.missing(missing);
+        }
+        if (order != null) {
+            builder.order(order);
+        }
+        if (sortMode != null) {
+            builder.sortMode(sortMode);
+        }
+        if (unmappedType != null) {
+            builder.unmappedType(unmappedType);
+        }
         return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
index e37eed61c6d..4a8bd6d1d9a 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
@@ -19,19 +19,35 @@
 
 package org.elasticsearch.search.sort;
 
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.BitSet;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.geo.GeoDistance;
+import org.elasticsearch.common.geo.GeoDistance.FixedSourceDistance;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
-import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
+import org.elasticsearch.index.fielddata.MultiGeoPointValues;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
@@ -44,27 +60,31 @@ import java.util.Objects;
 /**
  * A geo distance based sorting on a geo point like field.
  */
-public class GeoDistanceSortBuilder extends SortBuilder
-        implements ToXContent, NamedWriteable<GeoDistanceSortBuilder>, SortElementParserTemp<GeoDistanceSortBuilder> {
+public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder> {
     public static final String NAME = "_geo_distance";
+    public static final String ALTERNATIVE_NAME = "_geoDistance";
     public static final boolean DEFAULT_COERCE = false;
     public static final boolean DEFAULT_IGNORE_MALFORMED = false;
-
-    static final GeoDistanceSortBuilder PROTOTYPE = new GeoDistanceSortBuilder("", -1, -1);
+    public static final ParseField UNIT_FIELD = new ParseField("unit");
+    public static final ParseField REVERSE_FIELD = new ParseField("reverse");
+    public static final ParseField DISTANCE_TYPE_FIELD = new ParseField("distance_type");
+    public static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize");
+    public static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed");
+    public static final ParseField SORTMODE_FIELD = new ParseField("mode", "sort_mode");
+    public static final ParseField NESTED_PATH_FIELD = new ParseField("nested_path");
+    public static final ParseField NESTED_FILTER_FIELD = new ParseField("nested_filter");
 
     private final String fieldName;
     private final List<GeoPoint> points = new ArrayList<>();
 
     private GeoDistance geoDistance = GeoDistance.DEFAULT;
     private DistanceUnit unit = DistanceUnit.DEFAULT;
-    private SortOrder order = SortOrder.ASC;
-    
-    // TODO there is an enum that covers that parameter which we should be using here
-    private String sortMode = null;
+
+    private SortMode sortMode = null;
     @SuppressWarnings("rawtypes")
     private QueryBuilder nestedFilter;
     private String nestedPath;
-    
+
     // TODO switch to GeoValidationMethod enum
     private boolean coerce = DEFAULT_COERCE;
     private boolean ignoreMalformed = DEFAULT_IGNORE_MALFORMED;
@@ -109,7 +129,7 @@ public class GeoDistanceSortBuilder extends SortBuilder
         }
         this.fieldName = fieldName;
     }
-    
+
     /**
      * Copy constructor.
      * */
@@ -125,7 +145,38 @@ public class GeoDistanceSortBuilder extends SortBuilder
         this.coerce = original.coerce;
         this.ignoreMalformed = original.ignoreMalformed;
     }
-    
+
+    /**
+     * Read from a stream.
+     */
+    @SuppressWarnings("unchecked")
+    public GeoDistanceSortBuilder(StreamInput in) throws IOException {
+        fieldName = in.readString();
+        points.addAll((List<GeoPoint>) in.readGenericValue());
+        geoDistance = GeoDistance.readGeoDistanceFrom(in);
+        unit = DistanceUnit.readFromStream(in);
+        order = SortOrder.readFromStream(in);
+        sortMode = in.readOptionalWriteable(SortMode::readFromStream);
+        nestedFilter = in.readOptionalQuery();
+        nestedPath = in.readOptionalString();
+        coerce = in.readBoolean();
+        ignoreMalformed =in.readBoolean();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(fieldName);
+        out.writeGenericValue(points);
+        geoDistance.writeTo(out);
+        unit.writeTo(out);
+        order.writeTo(out);
+        out.writeOptionalWriteable(sortMode);
+        out.writeOptionalQuery(nestedFilter);
+        out.writeOptionalString(nestedPath);
+        out.writeBoolean(coerce);
+        out.writeBoolean(ignoreMalformed);
+    }
+
     /**
      * Returns the geo point like field the distance based sort operates on.
      * */
@@ -153,7 +204,7 @@ public class GeoDistanceSortBuilder extends SortBuilder
         this.points.addAll(Arrays.asList(points));
         return this;
     }
-    
+
     /**
      * Returns the points to create the range distance facets from.
      */
@@ -163,7 +214,7 @@ public class GeoDistanceSortBuilder extends SortBuilder
 
     /**
      * The geohash of the geo point to create the range distance facets from.
-     * 
+     *
      * Deprecated - please use points(GeoPoint... points) instead.
      */
     @Deprecated
@@ -173,7 +224,7 @@ public class GeoDistanceSortBuilder extends SortBuilder
         }
         return this;
     }
-    
+
     /**
      * The geo distance type used to compute the distance.
      */
@@ -181,7 +232,7 @@ public class GeoDistanceSortBuilder extends SortBuilder
         this.geoDistance = geoDistance;
         return this;
     }
-    
+
     /**
      * Returns the geo distance type used to compute the distance.
      */
@@ -204,37 +255,13 @@ public class GeoDistanceSortBuilder extends SortBuilder
         return this.unit;
     }
 
-    /**
-     * The order of sorting. Defaults to {@link SortOrder#ASC}.
-     */
-    @Override
-    public GeoDistanceSortBuilder order(SortOrder order) {
-        this.order = order;
-        return this;
-    }
-
-    /** Returns the order of sorting. */
-    public SortOrder order() { 
-        return this.order;
-    }
-
-    /**
-     * Not relevant.
-     *
-     * TODO should this throw an exception rather than silently ignore a parameter that is not used?
-     */
-    @Override
-    public GeoDistanceSortBuilder missing(Object missing) {
-        return this;
-    }
-
     /**
      * Defines which distance to use for sorting in the case a document contains multiple geo points.
      * Possible values: min and max
      */
-    public GeoDistanceSortBuilder sortMode(String sortMode) {
-        MultiValueMode temp = MultiValueMode.fromString(sortMode);
-        if (temp == MultiValueMode.SUM) {
+    public GeoDistanceSortBuilder sortMode(SortMode sortMode) {
+        Objects.requireNonNull(sortMode, "sort mode cannot be null");
+        if (sortMode == SortMode.SUM) {
             throw new IllegalArgumentException("sort_mode [sum] isn't supported for sorting by geo distance");
         }
         this.sortMode = sortMode;
@@ -242,7 +269,7 @@ public class GeoDistanceSortBuilder extends SortBuilder
     }
 
     /** Returns which distance to use for sorting in the case a document contains multiple geo points. */
-    public String sortMode() {
+    public SortMode sortMode() {
         return this.sortMode;
     }
 
@@ -250,16 +277,16 @@ public class GeoDistanceSortBuilder extends SortBuilder
      * Sets the nested filter that the nested objects should match with in order to be taken into account
      * for sorting.
      */
-    public GeoDistanceSortBuilder setNestedFilter(QueryBuilder nestedFilter) {
+    public GeoDistanceSortBuilder setNestedFilter(QueryBuilder<?> nestedFilter) {
         this.nestedFilter = nestedFilter;
         return this;
     }
 
-    /** 
+    /**
      * Returns the nested filter that the nested objects should match with in order to be taken into account
-     * for sorting. 
+     * for sorting.
      **/
-    public QueryBuilder getNestedFilter() {
+    public QueryBuilder<?> getNestedFilter() {
         return this.nestedFilter;
     }
 
@@ -271,7 +298,7 @@ public class GeoDistanceSortBuilder extends SortBuilder
         this.nestedPath = nestedPath;
         return this;
     }
-    
+
     /**
      * Returns the nested path if sorting occurs on a field that is inside a nested object. By default when sorting on a
      * field inside a nested object, the nearest upper nested object is selected as nested path.
@@ -295,13 +322,14 @@ public class GeoDistanceSortBuilder extends SortBuilder
         }
         return this;
     }
-    
+
     public boolean ignoreMalformed() {
         return this.ignoreMalformed;
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startObject(NAME);
 
         builder.startArray(fieldName);
@@ -310,27 +338,24 @@ public class GeoDistanceSortBuilder extends SortBuilder
         }
         builder.endArray();
 
-        builder.field("unit", unit);
-        builder.field("distance_type", geoDistance.name().toLowerCase(Locale.ROOT));
-        if (order == SortOrder.DESC) {
-            builder.field("reverse", true);
-        } else {
-            builder.field("reverse", false);
-        }
+        builder.field(UNIT_FIELD.getPreferredName(), unit);
+        builder.field(DISTANCE_TYPE_FIELD.getPreferredName(), geoDistance.name().toLowerCase(Locale.ROOT));
+        builder.field(ORDER_FIELD.getPreferredName(), order);
 
         if (sortMode != null) {
-            builder.field("mode", sortMode);
+            builder.field(SORTMODE_FIELD.getPreferredName(), sortMode);
         }
 
         if (nestedPath != null) {
-            builder.field("nested_path", nestedPath);
+            builder.field(NESTED_PATH_FIELD.getPreferredName(), nestedPath);
         }
         if (nestedFilter != null) {
-            builder.field("nested_filter", nestedFilter, params);
+            builder.field(NESTED_FILTER_FIELD.getPreferredName(), nestedFilter, params);
         }
-        builder.field("coerce", coerce);
-        builder.field("ignore_malformed", ignoreMalformed);
+        builder.field(COERCE_FIELD.getPreferredName(), coerce);
+        builder.field(IGNORE_MALFORMED_FIELD.getPreferredName(), ignoreMalformed);
 
+        builder.endObject();
         builder.endObject();
         return builder;
     }
@@ -369,59 +394,25 @@ public class GeoDistanceSortBuilder extends SortBuilder
                 this.unit, this.sortMode, this.order, this.nestedFilter, this.nestedPath, this.coerce, this.ignoreMalformed);
     }
 
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(fieldName);
-        out.writeGenericValue(points);
-        
-        geoDistance.writeTo(out);
-        unit.writeTo(out);
-        order.writeTo(out);
-        out.writeOptionalString(sortMode);
-        if (nestedFilter != null) {
-            out.writeBoolean(true);
-            out.writeQuery(nestedFilter);
-        } else {
-            out.writeBoolean(false);
-        }
-        out.writeOptionalString(nestedPath);
-        out.writeBoolean(coerce);
-        out.writeBoolean(ignoreMalformed);
-    }
-
-    @Override
-    public GeoDistanceSortBuilder readFrom(StreamInput in) throws IOException {
-        String fieldName = in.readString();
-        
-        ArrayList<GeoPoint> points = (ArrayList<GeoPoint>) in.readGenericValue(); 
-        GeoDistanceSortBuilder result = new GeoDistanceSortBuilder(fieldName, points.toArray(new GeoPoint[points.size()]));
-        
-        result.geoDistance(GeoDistance.readGeoDistanceFrom(in));
-        result.unit(DistanceUnit.readDistanceUnit(in));
-        result.order(SortOrder.readOrderFrom(in));
-        String sortMode = in.readOptionalString();
-        if (sortMode != null) {
-            result.sortMode(sortMode);
-        }
-        if (in.readBoolean()) {
-            result.setNestedFilter(in.readQuery());
-        }
-        result.setNestedPath(in.readOptionalString());
-        result.coerce(in.readBoolean());
-        result.ignoreMalformed(in.readBoolean());
-        return result;
-    }
-
-    @Override
-    public GeoDistanceSortBuilder fromXContent(QueryParseContext context, String elementName) throws IOException {
+    /**
+     * Creates a new {@link GeoDistanceSortBuilder} from the query held by the {@link QueryParseContext} in
+     * {@link org.elasticsearch.common.xcontent.XContent} format.
+     *
+     * @param context the input parse context. The state on the parser contained in this context will be changed as a side effect of this
+     *        method call
+     * @param elementName in some sort syntax variations the field name precedes the xContent object that specifies further parameters, e.g.
+     *        in '{ "foo": { "order" : "asc"} }'. When parsing the inner object, the field name can be passed in via this argument
+     */
+    public static GeoDistanceSortBuilder fromXContent(QueryParseContext context, String elementName) throws IOException {
         XContentParser parser = context.parser();
+        ParseFieldMatcher parseFieldMatcher = context.parseFieldMatcher();
         String fieldName = null;
         List<GeoPoint> geoPoints = new ArrayList<>();
         DistanceUnit unit = DistanceUnit.DEFAULT;
         GeoDistance geoDistance = GeoDistance.DEFAULT;
-        boolean reverse = false;
-        MultiValueMode sortMode = null;
-        QueryBuilder nestedFilter = null;
+        SortOrder order = SortOrder.ASC;
+        SortMode sortMode = null;
+        QueryBuilder<?> nestedFilter = null;
         String nestedPath = null;
 
         boolean coerce = GeoDistanceSortBuilder.DEFAULT_COERCE;
@@ -437,40 +428,37 @@ public class GeoDistanceSortBuilder extends SortBuilder
 
                 fieldName = currentName;
             } else if (token == XContentParser.Token.START_OBJECT) {
-                // the json in the format of -> field : { lat : 30, lon : 12 }
-                if ("nested_filter".equals(currentName) || "nestedFilter".equals(currentName)) {
-                    // TODO Note to remember: while this is kept as a QueryBuilder internally, 
-                    // we need to make sure to call toFilter() on it once on the shard 
-                    // (e.g. in the new build() method)
+                if (parseFieldMatcher.match(currentName, NESTED_FILTER_FIELD)) {
                     nestedFilter = context.parseInnerQueryBuilder();
                 } else {
+                    // the json in the format of -> field : { lat : 30, lon : 12 }
                     fieldName = currentName;
                     GeoPoint point = new GeoPoint();
                     GeoUtils.parseGeoPoint(parser, point);
                     geoPoints.add(point);
                 }
             } else if (token.isValue()) {
-                if ("reverse".equals(currentName)) {
-                    reverse = parser.booleanValue();
-                } else if ("order".equals(currentName)) {
-                    reverse = "desc".equals(parser.text());
-                } else if ("unit".equals(currentName)) {
+                if (parseFieldMatcher.match(currentName, REVERSE_FIELD)) {
+                    order = parser.booleanValue() ? SortOrder.DESC : SortOrder.ASC;
+                } else if (parseFieldMatcher.match(currentName, ORDER_FIELD)) {
+                    order = SortOrder.fromString(parser.text());
+                } else if (parseFieldMatcher.match(currentName, UNIT_FIELD)) {
                     unit = DistanceUnit.fromString(parser.text());
-                } else if ("distance_type".equals(currentName) || "distanceType".equals(currentName)) {
+                } else if (parseFieldMatcher.match(currentName, DISTANCE_TYPE_FIELD)) {
                     geoDistance = GeoDistance.fromString(parser.text());
-                } else if ("coerce".equals(currentName) || "normalize".equals(currentName)) {
+                } else if (parseFieldMatcher.match(currentName, COERCE_FIELD)) {
                     coerce = parser.booleanValue();
                     if (coerce == true) {
                         ignoreMalformed = true;
                     }
-                } else if ("ignore_malformed".equals(currentName)) {
+                } else if (parseFieldMatcher.match(currentName, IGNORE_MALFORMED_FIELD)) {
                     boolean ignore_malformed_value = parser.booleanValue();
                     if (coerce == false) {
                         ignoreMalformed = ignore_malformed_value;
                     }
-                } else if ("sort_mode".equals(currentName) || "sortMode".equals(currentName) || "mode".equals(currentName)) {
-                    sortMode = MultiValueMode.fromString(parser.text());
-                } else if ("nested_path".equals(currentName) || "nestedPath".equals(currentName)) {
+                } else if (parseFieldMatcher.match(currentName, SORTMODE_FIELD)) {
+                    sortMode = SortMode.fromString(parser.text());
+                } else if (parseFieldMatcher.match(currentName, NESTED_PATH_FIELD)) {
                     nestedPath = parser.text();
                 } else {
                     GeoPoint point = new GeoPoint();
@@ -484,20 +472,94 @@ public class GeoDistanceSortBuilder extends SortBuilder
         GeoDistanceSortBuilder result = new GeoDistanceSortBuilder(fieldName, geoPoints.toArray(new GeoPoint[geoPoints.size()]));
         result.geoDistance(geoDistance);
         result.unit(unit);
-        if (reverse) {
-            result.order(SortOrder.DESC);
-        } else {
-            result.order(SortOrder.ASC);
-        }
+        result.order(order);
         if (sortMode != null) {
-            result.sortMode(sortMode.name());
+            result.sortMode(sortMode);
         }
         result.setNestedFilter(nestedFilter);
         result.setNestedPath(nestedPath);
         result.coerce(coerce);
         result.ignoreMalformed(ignoreMalformed);
         return result;
+    }
 
+    @Override
+    public SortField build(QueryShardContext context) throws IOException {
+        final boolean indexCreatedBeforeV2_0 = context.indexVersionCreated().before(Version.V_2_0_0);
+        // validation was not available prior to 2.x, so to support bwc percolation queries we only ignore_malformed on 2.x created indexes
+        List<GeoPoint> localPoints = new ArrayList<GeoPoint>();
+        for (GeoPoint geoPoint : this.points) {
+            localPoints.add(new GeoPoint(geoPoint));
+        }
+
+        if (!indexCreatedBeforeV2_0 && !ignoreMalformed) {
+            for (GeoPoint point : localPoints) {
+                if (GeoUtils.isValidLatitude(point.lat()) == false) {
+                    throw new ElasticsearchParseException("illegal latitude value [{}] for [GeoDistanceSort]", point.lat());
+                }
+                if (GeoUtils.isValidLongitude(point.lon()) == false) {
+                    throw new ElasticsearchParseException("illegal longitude value [{}] for [GeoDistanceSort]", point.lon());
+                }
+            }
+        }
+
+        if (coerce) {
+            for (GeoPoint point : localPoints) {
+                GeoUtils.normalizePoint(point, coerce, coerce);
+            }
+        }
+
+        boolean reverse = (order == SortOrder.DESC);
+        final MultiValueMode finalSortMode;
+        if (sortMode == null) {
+            finalSortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN;
+        } else {
+            finalSortMode = MultiValueMode.fromString(sortMode.toString());
+        }
+
+        MappedFieldType fieldType = context.fieldMapper(fieldName);
+        if (fieldType == null) {
+            throw new IllegalArgumentException("failed to find mapper for [" + fieldName + "] for geo distance based sort");
+        }
+        final IndexGeoPointFieldData geoIndexFieldData = context.getForField(fieldType);
+        final FixedSourceDistance[] distances = new FixedSourceDistance[localPoints.size()];
+        for (int i = 0; i< localPoints.size(); i++) {
+            distances[i] = geoDistance.fixedSourceDistance(localPoints.get(i).lat(), localPoints.get(i).lon(), unit);
+        }
+
+        final Nested nested = resolveNested(context, nestedPath, nestedFilter);
+
+        IndexFieldData.XFieldComparatorSource geoDistanceComparatorSource = new IndexFieldData.XFieldComparatorSource() {
+
+            @Override
+            public SortField.Type reducedType() {
+                return SortField.Type.DOUBLE;
+            }
+
+            @Override
+            public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+                return new FieldComparator.DoubleComparator(numHits, null, null) {
+                    @Override
+                    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
+                        final MultiGeoPointValues geoPointValues = geoIndexFieldData.load(context).getGeoPointValues();
+                        final SortedNumericDoubleValues distanceValues = GeoDistance.distanceValues(geoPointValues, distances);
+                        final NumericDoubleValues selectedValues;
+                        if (nested == null) {
+                            selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE);
+                        } else {
+                            final BitSet rootDocs = nested.rootDocs(context);
+                            final DocIdSetIterator innerDocs = nested.innerDocs(context);
+                            selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE, rootDocs, innerDocs,
+                                    context.reader().maxDoc());
+                        }
+                        return selectedValues.getRawDoubleValues();
+                    }
+                };
+            }
+
+        };
+
+        return new SortField(fieldName, geoDistanceComparatorSource, reverse);
     }
 
     static void parseGeoPoints(XContentParser parser, List<GeoPoint> geoPoints) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java
deleted file mode 100644
index 27c8b8e0ed5..00000000000
--- a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.sort;
-
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.join.BitSetProducer;
-import org.apache.lucene.util.BitSet;
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
-import org.elasticsearch.common.geo.GeoDistance;
-import org.elasticsearch.common.geo.GeoDistance.FixedSourceDistance;
-import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.geo.GeoUtils;
-import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
-import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
-import org.elasticsearch.index.fielddata.MultiGeoPointValues;
-import org.elasticsearch.index.fielddata.NumericDoubleValues;
-import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.query.support.NestedInnerQueryParseSupport;
-import org.elasticsearch.search.MultiValueMode;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- *
- */
-public class GeoDistanceSortParser implements SortParser {
-
-    @Override
-    public String[] names() {
-        return new String[]{"_geo_distance", "_geoDistance"};
-    }
-
-    @Override
-    public SortField parse(XContentParser parser, SearchContext context) throws Exception {
-        String fieldName = null;
-        List<GeoPoint> geoPoints = new ArrayList<>();
-        DistanceUnit unit = DistanceUnit.DEFAULT;
-        GeoDistance geoDistance = GeoDistance.DEFAULT;
-        boolean reverse = false;
-        MultiValueMode sortMode = null;
-        NestedInnerQueryParseSupport nestedHelper = null;
-
-        final boolean indexCreatedBeforeV2_0 = context.indexShard().indexSettings().getIndexVersionCreated().before(Version.V_2_0_0);
-        boolean coerce = GeoDistanceSortBuilder.DEFAULT_COERCE;
-        boolean ignoreMalformed = GeoDistanceSortBuilder.DEFAULT_IGNORE_MALFORMED;
-
-        XContentParser.Token token;
-        String currentName = parser.currentName();
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentName = parser.currentName();
-            } else if (token == XContentParser.Token.START_ARRAY) {
-                GeoDistanceSortBuilder.parseGeoPoints(parser, geoPoints);
-
-                fieldName = currentName;
-            } else if (token == XContentParser.Token.START_OBJECT) {
-                // the json in the format of -> field : { lat : 30, lon : 12 }
-                if ("nested_filter".equals(currentName) || "nestedFilter".equals(currentName)) {
-                    if (nestedHelper == null) {
-                        nestedHelper = new NestedInnerQueryParseSupport(parser, context);
-                    }
-                    nestedHelper.filter();
-                } else {
-                    fieldName = currentName;
-                    GeoPoint point = new GeoPoint();
-                    GeoUtils.parseGeoPoint(parser, point);
-                    geoPoints.add(point);
-                }
-            } else if (token.isValue()) {
-                if ("reverse".equals(currentName)) {
-                    reverse = parser.booleanValue();
-                } else if ("order".equals(currentName)) {
-                    reverse = "desc".equals(parser.text());
-                } else if (currentName.equals("unit")) {
-                    unit = DistanceUnit.fromString(parser.text());
-                } else if (currentName.equals("distance_type") || currentName.equals("distanceType")) {
-                    geoDistance = GeoDistance.fromString(parser.text());
-                } else if ("coerce".equals(currentName) || (indexCreatedBeforeV2_0 && "normalize".equals(currentName))) {
-                    coerce = parser.booleanValue();
-                    if (coerce == true) {
-                        ignoreMalformed = true;
-                    }
-                } else if ("ignore_malformed".equals(currentName) && coerce == false) {
-                    ignoreMalformed = parser.booleanValue();
-                } else if ("sort_mode".equals(currentName) || "sortMode".equals(currentName) || "mode".equals(currentName)) {
-                    sortMode = MultiValueMode.fromString(parser.text());
-                } else if ("nested_path".equals(currentName) || "nestedPath".equals(currentName)) {
-                    if (nestedHelper == null) {
-                        nestedHelper = new NestedInnerQueryParseSupport(parser, context);
-                    }
-                    nestedHelper.setPath(parser.text());
-                } else {
-                    GeoPoint point = new GeoPoint();
-                    point.resetFromString(parser.text());
-                    geoPoints.add(point);
-                    fieldName = currentName;
-                }
-            }
-        }
-
-        // validation was not available prior to 2.x, so to support bwc percolation queries we only ignore_malformed on 2.x created indexes
-        if (!indexCreatedBeforeV2_0 && !ignoreMalformed) {
-            for (GeoPoint point : geoPoints) {
-                if (point.lat() > 90.0 || point.lat() < -90.0) {
-                    throw new ElasticsearchParseException("illegal latitude value [{}] for [GeoDistanceSort]", point.lat());
-                }
-                if (point.lon() > 180.0 || point.lon() < -180) {
-                    throw new ElasticsearchParseException("illegal longitude value [{}] for [GeoDistanceSort]", point.lon());
-                }
-            }
-        }
-
-        if (coerce) {
-            for (GeoPoint point : geoPoints) {
-                GeoUtils.normalizePoint(point, coerce, coerce);
-            }
-        }
-
-        if (sortMode == null) {
-            sortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN;
-        }
-
-        if (sortMode == MultiValueMode.SUM) {
-            throw new IllegalArgumentException("sort_mode [sum] isn't supported for sorting by geo distance");
-        }
-
-        MappedFieldType fieldType = context.smartNameFieldType(fieldName);
-        if (fieldType == null) {
-            throw new IllegalArgumentException("failed to find mapper for [" + fieldName + "] for geo distance based sort");
-        }
-        final MultiValueMode finalSortMode = sortMode; // final reference for use in the anonymous class
-        final IndexGeoPointFieldData geoIndexFieldData = context.fieldData().getForField(fieldType);
-        final FixedSourceDistance[] distances = new FixedSourceDistance[geoPoints.size()];
-        for (int i = 0; i< geoPoints.size(); i++) {
-            distances[i] = geoDistance.fixedSourceDistance(geoPoints.get(i).lat(), geoPoints.get(i).lon(), unit);
-        }
-
-        final Nested nested;
-        if (nestedHelper != null && nestedHelper.getPath() != null) {
-            BitSetProducer rootDocumentsFilter = context.bitsetFilterCache().getBitSetProducer(Queries.newNonNestedFilter());
-            Query innerDocumentsFilter;
-            if (nestedHelper.filterFound()) {
-                // TODO: use queries instead
-                innerDocumentsFilter = nestedHelper.getInnerFilter();
-            } else {
-                innerDocumentsFilter = nestedHelper.getNestedObjectMapper().nestedTypeFilter();
-            }
-            nested = new Nested(rootDocumentsFilter, context.searcher().createNormalizedWeight(innerDocumentsFilter, false));
-        } else {
-            nested = null;
-        }
-
-        IndexFieldData.XFieldComparatorSource geoDistanceComparatorSource = new IndexFieldData.XFieldComparatorSource() {
-
-            @Override
-            public SortField.Type reducedType() {
-                return SortField.Type.DOUBLE;
-            }
-
-            @Override
-            public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                return new FieldComparator.DoubleComparator(numHits, null, null) {
-                    @Override
-                    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
-                        final MultiGeoPointValues geoPointValues = geoIndexFieldData.load(context).getGeoPointValues();
-                        final SortedNumericDoubleValues distanceValues = GeoDistance.distanceValues(geoPointValues, distances);
-                        final NumericDoubleValues selectedValues;
-                        if (nested == null) {
-                            selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE);
-                        } else {
-                            final BitSet rootDocs = nested.rootDocs(context);
-                            final DocIdSetIterator innerDocs = nested.innerDocs(context);
-                            selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE, rootDocs, innerDocs, context.reader().maxDoc());
-                        }
-                        return selectedValues.getRawDoubleValues();
-                    }
-                };
-            }
-
-        };
-
-        return new SortField(fieldName, geoDistanceComparatorSource, reverse);
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java
index 7435ff95f45..271c94c59ad 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java
@@ -19,40 +19,126 @@
 
 package org.elasticsearch.search.sort;
 
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
+import java.util.Objects;
 
 /**
  * A sort builder allowing to sort by score.
- *
- *
  */
-public class ScoreSortBuilder extends SortBuilder {
+public class ScoreSortBuilder extends SortBuilder<ScoreSortBuilder> {
 
-    private SortOrder order;
+    public static final String NAME = "_score";
+    public static final ParseField REVERSE_FIELD = new ParseField("reverse");
+    public static final ParseField ORDER_FIELD = new ParseField("order");
+    private static final SortField SORT_SCORE = new SortField(null, SortField.Type.SCORE);
+    private static final SortField SORT_SCORE_REVERSE = new SortField(null, SortField.Type.SCORE, true);
 
     /**
-     * The order of sort scoring. By default, its {@link SortOrder#DESC}.
+     * Build a ScoreSortBuilder default to descending sort order.
      */
-    @Override
-    public ScoreSortBuilder order(SortOrder order) {
-        this.order = order;
-        return this;
+    public ScoreSortBuilder() {
+        // order defaults to desc when sorting on the _score
+        order(SortOrder.DESC);
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public ScoreSortBuilder(StreamInput in) throws IOException {
+        order(SortOrder.readFromStream(in));
     }
 
     @Override
-    public SortBuilder missing(Object missing) {
-        return this;
+    public void writeTo(StreamOutput out) throws IOException {
+        order.writeTo(out);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject("_score");
-        if (order == SortOrder.ASC) {
-            builder.field("reverse", true);
-        }
+        builder.startObject();
+        builder.startObject(NAME);
+        builder.field(ORDER_FIELD.getPreferredName(), order);
+        builder.endObject();
         builder.endObject();
         return builder;
     }
+
+    /**
+     * Creates a new {@link ScoreSortBuilder} from the query held by the {@link QueryParseContext} in
+     * {@link org.elasticsearch.common.xcontent.XContent} format.
+     *
+     * @param context the input parse context. The state on the parser contained in this context will be changed as a side effect of this
+     *        method call
+     * @param fieldName in some sort syntax variations the field name precedes the xContent object that specifies further parameters, e.g.
+     *        in '{ "foo": { "order" : "asc"} }'. When parsing the inner object, the field name can be passed in via this argument
+     */
+    public static ScoreSortBuilder fromXContent(QueryParseContext context, String fieldName) throws IOException {
+        XContentParser parser = context.parser();
+        ParseFieldMatcher matcher = context.parseFieldMatcher();
+
+        XContentParser.Token token;
+        String currentName = parser.currentName();
+        ScoreSortBuilder result = new ScoreSortBuilder();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentName = parser.currentName();
+            } else if (token.isValue()) {
+                if (matcher.match(currentName, REVERSE_FIELD)) {
+                    if (parser.booleanValue()) {
+                        result.order(SortOrder.ASC);
+                    }
+                    // else we keep the default DESC
+                } else if (matcher.match(currentName, ORDER_FIELD)) {
+                    result.order(SortOrder.fromString(parser.text()));
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] failed to parse field [" + currentName + "]");
+                }
+            } else {
+                throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] unexpected token [" + token + "]");
+            }
+        }
+        return result;
+    }
+
+    @Override
+    public SortField build(QueryShardContext context) {
+        if (order == SortOrder.DESC) {
+            return SORT_SCORE;
+        } else {
+            return SORT_SCORE_REVERSE;
+        }
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (this == object) {
+            return true;
+        }
+        if (object == null || getClass() != object.getClass()) {
+            return false;
+        }
+        ScoreSortBuilder other = (ScoreSortBuilder) object;
+        return Objects.equals(order, other.order);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(this.order);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
index e9a9c8df57c..2751d497519 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
@@ -19,26 +19,68 @@
 
 package org.elasticsearch.search.sort;
 
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.fielddata.FieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
+import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.Script.ScriptField;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.ScriptParameterParser;
+import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue;
+import org.elasticsearch.script.SearchScript;
+import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
 
 /**
  * Script sort builder allows to sort based on a custom script expression.
  */
-public class ScriptSortBuilder extends SortBuilder {
+public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> {
 
-    private Script script;
+    public static final String NAME = "_script";
+    public static final ParseField TYPE_FIELD = new ParseField("type");
+    public static final ParseField SCRIPT_FIELD = new ParseField("script");
+    public static final ParseField SORTMODE_FIELD = new ParseField("mode");
+    public static final ParseField NESTED_PATH_FIELD = new ParseField("nested_path");
+    public static final ParseField NESTED_FILTER_FIELD = new ParseField("nested_filter");
+    public static final ParseField PARAMS_FIELD = new ParseField("params");
 
-    private final String type;
+    private final Script script;
 
-    private SortOrder order;
+    private final ScriptSortType type;
 
-    private String sortMode;
+    private SortMode sortMode;
 
-    private QueryBuilder nestedFilter;
+    private QueryBuilder<?> nestedFilter;
 
     private String nestedPath;
 
@@ -47,47 +89,99 @@ public class ScriptSortBuilder extends SortBuilder {
      *
      * @param script
      *            The script to use.
+     * @param type
+     *            The type of the script, can be either {@link ScriptSortType#STRING} or
+     *            {@link ScriptSortType#NUMBER}
      */
-    public ScriptSortBuilder(Script script, String type) {
+    public ScriptSortBuilder(Script script, ScriptSortType type) {
+        Objects.requireNonNull(script, "script cannot be null");
+        Objects.requireNonNull(type, "type cannot be null");
         this.script = script;
         this.type = type;
     }
 
-    /**
-     * Sets the sort order.
-     */
-    @Override
-    public ScriptSortBuilder order(SortOrder order) {
-        this.order = order;
-        return this;
+    ScriptSortBuilder(ScriptSortBuilder original) {
+        this.script = original.script;
+        this.type = original.type;
+        this.order = original.order;
+        this.sortMode = original.sortMode;
+        this.nestedFilter = original.nestedFilter;
+        this.nestedPath = original.nestedPath;
     }
 
     /**
-     * Not really relevant.
+     * Read from a stream.
      */
+    public ScriptSortBuilder(StreamInput in) throws IOException {
+        script = Script.readScript(in);
+        type = ScriptSortType.readFromStream(in);
+        order = SortOrder.readFromStream(in);
+        sortMode = in.readOptionalWriteable(SortMode::readFromStream);
+        nestedPath = in.readOptionalString();
+        nestedFilter = in.readOptionalQuery();
+    }
+
     @Override
-    public SortBuilder missing(Object missing) {
-        return this;
+    public void writeTo(StreamOutput out) throws IOException {
+        script.writeTo(out);
+        type.writeTo(out);
+        order.writeTo(out);
+        out.writeOptionalWriteable(sortMode);
+        out.writeOptionalString(nestedPath);
+        out.writeOptionalQuery(nestedFilter);
     }
 
     /**
-     * Defines which distance to use for sorting in the case a document contains multiple geo points.
-     * Possible values: min and max
+     * Get the script used in this sort.
      */
-    public ScriptSortBuilder sortMode(String sortMode) {
+    public Script script() {
+        return this.script;
+    }
+
+    /**
+     * Get the type used in this sort.
+     */
+    public ScriptSortType type() {
+        return this.type;
+    }
+
+    /**
+     * Defines which distance to use for sorting in the case a document contains multiple values.<br>
+     * For {@link ScriptSortType#STRING}, the set of possible values is restricted to {@link SortMode#MIN} and {@link SortMode#MAX}
+     */
+    public ScriptSortBuilder sortMode(SortMode sortMode) {
+        Objects.requireNonNull(sortMode, "sort mode cannot be null.");
+        if (ScriptSortType.STRING.equals(type) && (sortMode == SortMode.SUM || sortMode == SortMode.AVG ||
+                sortMode == SortMode.MEDIAN)) {
+            throw new IllegalArgumentException("script sort of type [string] doesn't support mode [" + sortMode + "]");
+        }
         this.sortMode = sortMode;
         return this;
     }
 
+    /**
+     * Get the sort mode.
+     */
+    public SortMode sortMode() {
+        return this.sortMode;
+    }
+
     /**
      * Sets the nested filter that the nested objects should match with in order to be taken into account
      * for sorting.
      */
-    public ScriptSortBuilder setNestedFilter(QueryBuilder nestedFilter) {
+    public ScriptSortBuilder setNestedFilter(QueryBuilder<?> nestedFilter) {
         this.nestedFilter = nestedFilter;
         return this;
     }
 
+    /**
+     * Gets the nested filter.
+     */
+    public QueryBuilder<?> getNestedFilter() {
+        return this.nestedFilter;
+    }
+
     /**
      * Sets the nested path if sorting occurs on a field that is inside a nested object. For sorting by script this
      * needs to be specified.
@@ -97,24 +191,248 @@ public class ScriptSortBuilder extends SortBuilder {
         return this;
     }
 
+    /**
+     * Gets the nested path.
+     */
+    public String getNestedPath() {
+        return this.nestedPath;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params builderParams) throws IOException {
-        builder.startObject("_script");
-        builder.field("script", script);
-        builder.field("type", type);
-        if (order == SortOrder.DESC) {
-            builder.field("reverse", true);
-        }
+        builder.startObject();
+        builder.startObject(NAME);
+        builder.field(SCRIPT_FIELD.getPreferredName(), script);
+        builder.field(TYPE_FIELD.getPreferredName(), type);
+        builder.field(ORDER_FIELD.getPreferredName(), order);
         if (sortMode != null) {
-            builder.field("mode", sortMode);
+            builder.field(SORTMODE_FIELD.getPreferredName(), sortMode);
         }
         if (nestedPath != null) {
-            builder.field("nested_path", nestedPath);
+            builder.field(NESTED_PATH_FIELD.getPreferredName(), nestedPath);
         }
         if (nestedFilter != null) {
-            builder.field("nested_filter", nestedFilter, builderParams);
+            builder.field(NESTED_FILTER_FIELD.getPreferredName(), nestedFilter, builderParams);
         }
         builder.endObject();
+        builder.endObject();
         return builder;
     }
+
+    /**
+     * Creates a new {@link ScriptSortBuilder} from the query held by the {@link QueryParseContext} in
+     * {@link org.elasticsearch.common.xcontent.XContent} format.
+     *
+     * @param context the input parse context. The state on the parser contained in this context will be changed as a side effect of this
+     *        method call
+     * @param elementName in some sort syntax variations the field name precedes the xContent object that specifies further parameters, e.g.
+     *        in '{ "foo": { "order" : "asc"} }'. When parsing the inner object, the field name can be passed in via this argument
+     */
+    public static ScriptSortBuilder fromXContent(QueryParseContext context, String elementName) throws IOException {
+        ScriptParameterParser scriptParameterParser = new ScriptParameterParser();
+        XContentParser parser = context.parser();
+        ParseFieldMatcher parseField = context.parseFieldMatcher();
+        Script script = null;
+        ScriptSortType type = null;
+        SortMode sortMode = null;
+        SortOrder order = null;
+        QueryBuilder<?> nestedFilter = null;
+        String nestedPath = null;
+        Map<String, Object> params = new HashMap<>();
+
+        XContentParser.Token token;
+        String currentName = parser.currentName();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentName = parser.currentName();
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (parseField.match(currentName, ScriptField.SCRIPT)) {
+                    script = Script.parse(parser, parseField);
+                } else if (parseField.match(currentName, PARAMS_FIELD)) {
+                    params = parser.map();
+                } else if (parseField.match(currentName, NESTED_FILTER_FIELD)) {
+                    nestedFilter = context.parseInnerQueryBuilder();
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] failed to parse field [" + currentName + "]");
+                }
+            } else if (token.isValue()) {
+                if (parseField.match(currentName, ORDER_FIELD)) {
+                    order = SortOrder.fromString(parser.text());
+                } else if (scriptParameterParser.token(currentName, token, parser, parseField)) {
+                    // Do Nothing (handled by ScriptParameterParser
+                } else if (parseField.match(currentName, TYPE_FIELD)) {
+                    type = ScriptSortType.fromString(parser.text());
+                } else if (parseField.match(currentName, SORTMODE_FIELD)) {
+                    sortMode = SortMode.fromString(parser.text());
+                } else if (parseField.match(currentName, NESTED_PATH_FIELD)) {
+                    nestedPath = parser.text();
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] failed to parse field [" + currentName + "]");
+                }
+            } else {
+                throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] unexpected token [" + token + "]");
+            }
+        }
+
+        if (script == null) { // Didn't find anything using the new API so try using the old one instead
+            ScriptParameterValue scriptValue = scriptParameterParser.getDefaultScriptParameterValue();
+            if (scriptValue != null) {
+                if (params == null) {
+                    params = new HashMap<>();
+                }
+                script = new Script(scriptValue.script(), scriptValue.scriptType(), scriptParameterParser.lang(), params);
+            }
+        }
+
+        ScriptSortBuilder result = new ScriptSortBuilder(script, type);
+        if (order != null) {
+            result.order(order);
+        }
+        if (sortMode != null) {
+            result.sortMode(sortMode);
+        }
+        if (nestedFilter != null) {
+            result.setNestedFilter(nestedFilter);
+        }
+        if (nestedPath != null) {
+            result.setNestedPath(nestedPath);
+        }
+        return result;
+    }
+
+
+    @Override
+    public SortField build(QueryShardContext context) throws IOException {
+        final SearchScript searchScript = context.getScriptService().search(
+                context.lookup(), script, ScriptContext.Standard.SEARCH, Collections.emptyMap());
+
+        MultiValueMode valueMode = null;
+        if (sortMode != null) {
+            valueMode = MultiValueMode.fromString(sortMode.toString());
+        }
+        boolean reverse = (order == SortOrder.DESC);
+        if (valueMode == null) {
+            valueMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN;
+        }
+
+        final Nested nested = resolveNested(context, nestedPath, nestedFilter);
+        final IndexFieldData.XFieldComparatorSource fieldComparatorSource;
+        switch (type) {
+            case STRING:
+                fieldComparatorSource = new BytesRefFieldComparatorSource(null, null, valueMode, nested) {
+                    LeafSearchScript leafScript;
+                    @Override
+                    protected SortedBinaryDocValues getValues(LeafReaderContext context) throws IOException {
+                        leafScript = searchScript.getLeafSearchScript(context);
+                        final BinaryDocValues values = new BinaryDocValues() {
+                            final BytesRefBuilder spare = new BytesRefBuilder();
+                            @Override
+                            public BytesRef get(int docID) {
+                                leafScript.setDocument(docID);
+                                spare.copyChars(leafScript.run().toString());
+                                return spare.get();
+                            }
+                        };
+                        return FieldData.singleton(values, null);
+                    }
+                    @Override
+                    protected void setScorer(Scorer scorer) {
+                        leafScript.setScorer(scorer);
+                    }
+                };
+                break;
+            case NUMBER:
+                fieldComparatorSource = new DoubleValuesComparatorSource(null, Double.MAX_VALUE, valueMode, nested) {
+                    LeafSearchScript leafScript;
+                    @Override
+                    protected SortedNumericDoubleValues getValues(LeafReaderContext context) throws IOException {
+                        leafScript = searchScript.getLeafSearchScript(context);
+                        final NumericDoubleValues values = new NumericDoubleValues() {
+                            @Override
+                            public double get(int docID) {
+                                leafScript.setDocument(docID);
+                                return leafScript.runAsDouble();
+                            }
+                        };
+                        return FieldData.singleton(values, null);
+                    }
+                    @Override
+                    protected void setScorer(Scorer scorer) {
+                        leafScript.setScorer(scorer);
+                    }
+                };
+                break;
+            default:
+            throw new QueryShardException(context, "custom script sort type [" + type + "] not supported");
+        }
+
+        return new SortField("_script", fieldComparatorSource, reverse);
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (this == object) {
+            return true;
+        }
+        if (object == null || getClass() != object.getClass()) {
+            return false;
+        }
+        ScriptSortBuilder other = (ScriptSortBuilder) object;
+        return Objects.equals(script, other.script) &&
+                Objects.equals(type, other.type) &&
+                Objects.equals(order, other.order) &&
+                Objects.equals(sortMode, other.sortMode) &&
+                Objects.equals(nestedFilter, other.nestedFilter) &&
+                Objects.equals(nestedPath, other.nestedPath);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(script, type, order, sortMode, nestedFilter, nestedPath);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    public enum ScriptSortType implements Writeable<ScriptSortType> {
+        /** script sort for a string value **/
+        STRING,
+        /** script sort for a numeric value **/
+        NUMBER;
+
+        @Override
+        public void writeTo(final StreamOutput out) throws IOException {
+            out.writeVInt(ordinal());
+        }
+
+        /**
+         * Read from a stream.
+         */
+        static ScriptSortType readFromStream(final StreamInput in) throws IOException {
+            int ordinal = in.readVInt();
+            if (ordinal < 0 || ordinal >= values().length) {
+                throw new IOException("Unknown ScriptSortType ordinal [" + ordinal + "]");
+            }
+            return values()[ordinal];
+        }
+
+        public static ScriptSortType fromString(final String str) {
+            Objects.requireNonNull(str, "input string is null");
+            switch (str.toLowerCase(Locale.ROOT)) {
+                case ("string"):
+                    return ScriptSortType.STRING;
+                case ("number"):
+                    return ScriptSortType.NUMBER;
+                default:
+                    throw new IllegalArgumentException("Unknown ScriptSortType [" + str + "]");
+            }
+        }
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java
deleted file mode 100644
index e4fe2c08f75..00000000000
--- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.sort;
-
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.join.BitSetProducer;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.FieldData;
-import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
-import org.elasticsearch.index.fielddata.NumericDoubleValues;
-import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
-import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
-import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
-import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
-import org.elasticsearch.index.query.support.NestedInnerQueryParseSupport;
-import org.elasticsearch.script.LeafSearchScript;
-import org.elasticsearch.script.Script;
-import org.elasticsearch.script.Script.ScriptField;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptParameterParser;
-import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue;
-import org.elasticsearch.script.SearchScript;
-import org.elasticsearch.search.MultiValueMode;
-import org.elasticsearch.search.SearchParseException;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- *
- */
-public class ScriptSortParser implements SortParser {
-
-    private static final String STRING_SORT_TYPE = "string";
-    private static final String NUMBER_SORT_TYPE = "number";
-
-    @Override
-    public String[] names() {
-        return new String[]{"_script"};
-    }
-
-    @Override
-    public SortField parse(XContentParser parser, SearchContext context) throws Exception {
-        ScriptParameterParser scriptParameterParser = new ScriptParameterParser();
-        Script script = null;
-        String type = null;
-        Map<String, Object> params = null;
-        boolean reverse = false;
-        MultiValueMode sortMode = null;
-        NestedInnerQueryParseSupport nestedHelper = null;
-
-        XContentParser.Token token;
-        String currentName = parser.currentName();
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentName = parser.currentName();
-            } else if (token == XContentParser.Token.START_OBJECT) {
-                if (context.parseFieldMatcher().match(currentName, ScriptField.SCRIPT)) {
-                    script = Script.parse(parser, context.parseFieldMatcher());
-                } else if ("params".equals(currentName)) {
-                    params = parser.map();
-                } else if ("nested_filter".equals(currentName) || "nestedFilter".equals(currentName)) {
-                    if (nestedHelper == null) {
-                        nestedHelper = new NestedInnerQueryParseSupport(parser, context);
-                    }
-                    nestedHelper.filter();
-                }
-            } else if (token.isValue()) {
-                if ("reverse".equals(currentName)) {
-                    reverse = parser.booleanValue();
-                } else if ("order".equals(currentName)) {
-                    reverse = "desc".equals(parser.text());
-                } else if (scriptParameterParser.token(currentName, token, parser, context.parseFieldMatcher())) {
-                    // Do Nothing (handled by ScriptParameterParser
-                } else if ("type".equals(currentName)) {
-                    type = parser.text();
-                } else if ("mode".equals(currentName)) {
-                    sortMode = MultiValueMode.fromString(parser.text());
-                } else if ("nested_path".equals(currentName) || "nestedPath".equals(currentName)) {
-                    if (nestedHelper == null) {
-                        nestedHelper = new NestedInnerQueryParseSupport(parser, context);
-                    }
-                    nestedHelper.setPath(parser.text());
-                }
-            }
-        }
-
-        if (script == null) { // Didn't find anything using the new API so try using the old one instead
-            ScriptParameterValue scriptValue = scriptParameterParser.getDefaultScriptParameterValue();
-            if (scriptValue != null) {
-                if (params == null) {
-                    params = new HashMap<>();
-                }
-                script = new Script(scriptValue.script(), scriptValue.scriptType(), scriptParameterParser.lang(), params);
-            }
-        } else if (params != null) {
-            throw new SearchParseException(context, "script params must be specified inside script object", parser.getTokenLocation());
-        }
-
-        if (script == null) {
-            throw new SearchParseException(context, "_script sorting requires setting the script to sort by", parser.getTokenLocation());
-        }
-        if (type == null) {
-            throw new SearchParseException(context, "_script sorting requires setting the type of the script", parser.getTokenLocation());
-        }
-        final SearchScript searchScript = context.scriptService().search(context.lookup(), script, ScriptContext.Standard.SEARCH, Collections.emptyMap());
-
-        if (STRING_SORT_TYPE.equals(type) && (sortMode == MultiValueMode.SUM || sortMode == MultiValueMode.AVG)) {
-            throw new SearchParseException(context, "type [string] doesn't support mode [" + sortMode + "]", parser.getTokenLocation());
-        }
-
-        if (sortMode == null) {
-            sortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN;
-        }
-
-        // If nested_path is specified, then wrap the `fieldComparatorSource` in a `NestedFieldComparatorSource`
-        final Nested nested;
-        if (nestedHelper != null && nestedHelper.getPath() != null) {
-            BitSetProducer rootDocumentsFilter = context.bitsetFilterCache().getBitSetProducer(Queries.newNonNestedFilter());
-            Query innerDocumentsFilter;
-            if (nestedHelper.filterFound()) {
-                // TODO: use queries instead
-                innerDocumentsFilter = nestedHelper.getInnerFilter();
-            } else {
-                innerDocumentsFilter = nestedHelper.getNestedObjectMapper().nestedTypeFilter();
-            }
-            nested = new Nested(rootDocumentsFilter, context.searcher().createNormalizedWeight(innerDocumentsFilter, false));
-        } else {
-            nested = null;
-        }
-
-        final IndexFieldData.XFieldComparatorSource fieldComparatorSource;
-        switch (type) {
-            case STRING_SORT_TYPE:
-                fieldComparatorSource = new BytesRefFieldComparatorSource(null, null, sortMode, nested) {
-                    LeafSearchScript leafScript;
-                    @Override
-                    protected SortedBinaryDocValues getValues(LeafReaderContext context) throws IOException {
-                        leafScript = searchScript.getLeafSearchScript(context);
-                        final BinaryDocValues values = new BinaryDocValues() {
-                            final BytesRefBuilder spare = new BytesRefBuilder();
-                            @Override
-                            public BytesRef get(int docID) {
-                                leafScript.setDocument(docID);
-                                spare.copyChars(leafScript.run().toString());
-                                return spare.get();
-                            }
-                        };
-                        return FieldData.singleton(values, null);
-                    }
-                    @Override
-                    protected void setScorer(Scorer scorer) {
-                        leafScript.setScorer(scorer);
-                    }
-                };
-                break;
-            case NUMBER_SORT_TYPE:
-                // TODO: should we rather sort missing values last?
-                fieldComparatorSource = new DoubleValuesComparatorSource(null, Double.MAX_VALUE, sortMode, nested) {
-                    LeafSearchScript leafScript;
-                    @Override
-                    protected SortedNumericDoubleValues getValues(LeafReaderContext context) throws IOException {
-                        leafScript = searchScript.getLeafSearchScript(context);
-                        final NumericDoubleValues values = new NumericDoubleValues() {
-                            @Override
-                            public double get(int docID) {
-                                leafScript.setDocument(docID);
-                                return leafScript.runAsDouble();
-                            }
-                        };
-                        return FieldData.singleton(values, null);
-                    }
-                    @Override
-                    protected void setScorer(Scorer scorer) {
-                        leafScript.setScorer(scorer);
-                    }
-                };
-                break;
-            default:
-            throw new SearchParseException(context, "custom script sort type [" + type + "] not supported", parser.getTokenLocation());
-        }
-
-        return new SortField("_script", fieldComparatorSource, reverse);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
index da80506dde2..ee6af01c93b 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
@@ -19,36 +19,191 @@
 
 package org.elasticsearch.search.sort;
 
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.join.BitSetProducer;
+import org.elasticsearch.action.support.ToXContentToBytes;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
+import org.elasticsearch.index.mapper.object.ObjectMapper;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+
+import static java.util.Collections.unmodifiableMap;
 
 /**
  *
  */
-public abstract class SortBuilder implements ToXContent {
+public abstract class SortBuilder<T extends SortBuilder<?>> extends ToXContentToBytes implements NamedWriteable<T> {
 
-    @Override
-    public String toString() {
-        try {
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.prettyPrint();
-            toXContent(builder, EMPTY_PARAMS);
-            return builder.string();
-        } catch (Exception e) {
-            throw new ElasticsearchException("Failed to build query", e);
-        }
+    protected SortOrder order = SortOrder.ASC;
+    public static final ParseField ORDER_FIELD = new ParseField("order");
+
+    private static final Map<String, Parser<?>> PARSERS;
+    static {
+        Map<String, Parser<?>> parsers = new HashMap<>();
+        parsers.put(ScriptSortBuilder.NAME, ScriptSortBuilder::fromXContent);
+        parsers.put(GeoDistanceSortBuilder.NAME, GeoDistanceSortBuilder::fromXContent);
+        parsers.put(GeoDistanceSortBuilder.ALTERNATIVE_NAME, GeoDistanceSortBuilder::fromXContent);
+        parsers.put(ScoreSortBuilder.NAME, ScoreSortBuilder::fromXContent);
+        // FieldSortBuilder gets involved if the user specifies a name that isn't one of these.
+        PARSERS = unmodifiableMap(parsers);
     }
 
     /**
-     * The order of sorting. Defaults to {@link SortOrder#ASC}.
+     * Create a @link {@link SortField} from this builder.
      */
-    public abstract SortBuilder order(SortOrder order);
+    protected abstract SortField build(QueryShardContext context) throws IOException;
 
     /**
-     * Sets the value when a field is missing in a doc. Can also be set to <tt>_last</tt> or
-     * <tt>_first</tt> to sort missing last or first respectively.
+     * Set the order of sorting.
      */
-    public abstract SortBuilder missing(Object missing);
+    @SuppressWarnings("unchecked")
+    public T order(SortOrder order) {
+        Objects.requireNonNull(order, "sort order cannot be null.");
+        this.order = order;
+        return (T) this;
+    }
+
+    /**
+     * Return the {@link SortOrder} used for this {@link SortBuilder}.
+     */
+    public SortOrder order() {
+        return this.order;
+    }
+
+    public static List<SortBuilder<?>> fromXContent(QueryParseContext context) throws IOException {
+        List<SortBuilder<?>> sortFields = new ArrayList<>(2);
+        XContentParser parser = context.parser();
+        XContentParser.Token token = parser.currentToken();
+        if (token == XContentParser.Token.START_ARRAY) {
+            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                if (token == XContentParser.Token.START_OBJECT) {
+                    parseCompoundSortField(parser, context, sortFields);
+                } else if (token == XContentParser.Token.VALUE_STRING) {
+                    String fieldName = parser.text();
+                    sortFields.add(fieldOrScoreSort(fieldName));
+                } else {
+                    throw new IllegalArgumentException("malformed sort format, "
+                            + "within the sort array, an object, or an actual string are allowed");
+                }
+            }
+        } else if (token == XContentParser.Token.VALUE_STRING) {
+            String fieldName = parser.text();
+            sortFields.add(fieldOrScoreSort(fieldName));
+        } else if (token == XContentParser.Token.START_OBJECT) {
+            parseCompoundSortField(parser, context, sortFields);
+        } else {
+            throw new IllegalArgumentException("malformed sort format, either start with array, object, or an actual string");
+        }
+        return sortFields;
+    }
+
+    private static SortBuilder<?> fieldOrScoreSort(String fieldName) {
+        if (fieldName.equals(ScoreSortBuilder.NAME)) {
+            return new ScoreSortBuilder();
+        } else {
+            return new FieldSortBuilder(fieldName);
+        }
+    }
+
+    private static void parseCompoundSortField(XContentParser parser, QueryParseContext context, List<SortBuilder<?>> sortFields)
+            throws IOException {
+        XContentParser.Token token;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                String fieldName = parser.currentName();
+                token = parser.nextToken();
+                if (token == XContentParser.Token.VALUE_STRING) {
+                    SortOrder order = SortOrder.fromString(parser.text());
+                    sortFields.add(fieldOrScoreSort(fieldName).order(order));
+                } else {
+                    if (PARSERS.containsKey(fieldName)) {
+                        sortFields.add(PARSERS.get(fieldName).fromXContent(context, fieldName));
+                    } else {
+                        sortFields.add(FieldSortBuilder.fromXContent(context, fieldName));
+                    }
+                }
+            }
+        }
+    }
+
+    public static void parseSort(XContentParser parser, SearchContext context) throws IOException {
+        QueryParseContext parseContext = context.getQueryShardContext().parseContext();
+        parseContext.reset(parser);
+        Optional<Sort> sortOptional = buildSort(SortBuilder.fromXContent(parseContext), context.getQueryShardContext());
+        if (sortOptional.isPresent()) {
+            context.sort(sortOptional.get());
+        }
+    }
+
+    public static Optional<Sort> buildSort(List<SortBuilder<?>> sortBuilders, QueryShardContext context) throws IOException {
+        List<SortField> sortFields = new ArrayList<>(sortBuilders.size());
+        for (SortBuilder<?> builder : sortBuilders) {
+            sortFields.add(builder.build(context));
+        }
+        if (!sortFields.isEmpty()) {
+            // optimize if we just sort on score non reversed, we don't really
+            // need sorting
+            boolean sort;
+            if (sortFields.size() > 1) {
+                sort = true;
+            } else {
+                SortField sortField = sortFields.get(0);
+                if (sortField.getType() == SortField.Type.SCORE && !sortField.getReverse()) {
+                    sort = false;
+                } else {
+                    sort = true;
+                }
+            }
+            if (sort) {
+                return Optional.of(new Sort(sortFields.toArray(new SortField[sortFields.size()])));
+            }
+        }
+        return Optional.empty();
+    }
+
+    protected static Nested resolveNested(QueryShardContext context, String nestedPath, QueryBuilder<?> nestedFilter) throws IOException {
+        Nested nested = null;
+        if (nestedPath != null) {
+            BitSetProducer rootDocumentsFilter = context.bitsetFilter(Queries.newNonNestedFilter());
+            ObjectMapper nestedObjectMapper = context.getObjectMapper(nestedPath);
+            if (nestedObjectMapper == null) {
+                throw new QueryShardException(context, "[nested] failed to find nested object under path [" + nestedPath + "]");
+            }
+            if (!nestedObjectMapper.nested().isNested()) {
+                throw new QueryShardException(context, "[nested] nested object under path [" + nestedPath + "] is not of nested type");
+            }
+            Query innerDocumentsQuery;
+            if (nestedFilter != null) {
+                context.nestedScope().nextLevel(nestedObjectMapper);
+                innerDocumentsQuery = QueryBuilder.rewriteQuery(nestedFilter, context).toFilter(context);
+                context.nestedScope().previousLevel();
+            } else {
+                innerDocumentsQuery = nestedObjectMapper.nestedTypeFilter();
+            }
+            nested = new Nested(rootDocumentsFilter,  innerDocumentsQuery);
+        }
+        return nested;
+    }
+
+    @FunctionalInterface
+    private interface Parser<T extends SortBuilder<?>> {
+        T fromXContent(QueryParseContext context, String elementName) throws IOException;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortBuilders.java b/core/src/main/java/org/elasticsearch/search/sort/SortBuilders.java
index f326fee3837..3eae9b8d019 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/SortBuilders.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/SortBuilders.java
@@ -21,8 +21,7 @@ package org.elasticsearch.search.sort;
 
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.script.Script;
-
-import java.util.Arrays;
+import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
 
 /**
  * A set of static factory methods for {@link SortBuilder}s.
@@ -53,7 +52,7 @@ public class SortBuilders {
      * @param script The script to use.
      * @param type   The type, can either be "string" or "number".
      */
-    public static ScriptSortBuilder scriptSort(Script script, String type) {
+    public static ScriptSortBuilder scriptSort(Script script, ScriptSortType type) {
         return new ScriptSortBuilder(script, type);
     }
 
@@ -63,12 +62,12 @@ public class SortBuilders {
      * @param fieldName The geo point like field name.
      * @param lat Latitude of the point to create the range distance facets from.
      * @param lon Longitude of the point to create the range distance facets from.
-     * 
+     *
      */
     public static GeoDistanceSortBuilder geoDistanceSort(String fieldName, double lat, double lon) {
         return new GeoDistanceSortBuilder(fieldName, lat, lon);
     }
-    
+
     /**
      * Constructs a new distance based sort on a geo point like field.
      *
@@ -87,5 +86,5 @@ public class SortBuilders {
      */
     public static GeoDistanceSortBuilder geoDistanceSort(String fieldName, String ... geohashes) {
         return new GeoDistanceSortBuilder(fieldName, geohashes);
-    }    
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortElementParserTemp.java b/core/src/main/java/org/elasticsearch/search/sort/SortElementParserTemp.java
deleted file mode 100644
index 8893471b6c1..00000000000
--- a/core/src/main/java/org/elasticsearch/search/sort/SortElementParserTemp.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.sort;
-
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.index.query.QueryParseContext;
-
-import java.io.IOException;
-
-// TODO once sort refactoring is done this needs to be merged into SortBuilder
-public interface SortElementParserTemp<T extends ToXContent> {
-    /**
-     * Creates a new SortBuilder from the json held by the {@link SortElementParserTemp}
-     * in {@link org.elasticsearch.common.xcontent.XContent} format
-     *
-     * @param context
-     *            the input parse context. The state on the parser contained in
-     *            this context will be changed as a side effect of this method
-     *            call
-     * @return the new item
-     */
-    T fromXContent(QueryParseContext context, String elementName) throws IOException;
-}
diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortMode.java b/core/src/main/java/org/elasticsearch/search/sort/SortMode.java
new file mode 100644
index 00000000000..c6b3e1b10b4
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/sort/SortMode.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.sort;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Objects;
+
+/**
+ * Elasticsearch supports sorting by array or multi-valued fields. The SortMode option controls what array value is picked
+ * for sorting the document it belongs to. The mode option can have the following values:
+ * <ul>
+ * <li>min - Pick the lowest value.</li>
+ * <li>max - Pick the highest value.</li>
+ * <li>sum - Use the sum of all values as sort value. Only applicable for number based array fields.</li>
+ * <li>avg - Use the average of all values as sort value. Only applicable for number based array fields.</li>
+ * <li>median - Use the median of all values as sort value. Only applicable for number based array fields.</li>
+ * </ul>
+ */
+public enum SortMode implements Writeable<SortMode> {
+    /** pick the lowest value **/
+    MIN,
+    /** pick the highest value **/
+    MAX,
+    /** Use the sum of all values as sort value. Only applicable for number based array fields. **/
+    SUM,
+    /** Use the average of all values as sort value. Only applicable for number based array fields. **/
+    AVG,
+    /** Use the median of all values as sort value. Only applicable for number based array fields. **/
+    MEDIAN;
+
+    @Override
+    public void writeTo(final StreamOutput out) throws IOException {
+        out.writeVInt(ordinal());
+    }
+
+    public static SortMode readFromStream(StreamInput in) throws IOException {
+        int ordinal = in.readVInt();
+        if (ordinal < 0 || ordinal >= values().length) {
+            throw new IOException("Unknown SortMode ordinal [" + ordinal + "]");
+        }
+        return values()[ordinal];
+    }
+
+    public static SortMode fromString(final String str) {
+        Objects.requireNonNull(str, "input string is null");
+        switch (str.toLowerCase(Locale.ROOT)) {
+            case ("min"):
+                return MIN;
+            case ("max"):
+                return MAX;
+            case ("sum"):
+                return SUM;
+            case ("avg"):
+                return AVG;
+            case ("median"):
+                return MEDIAN;
+            default:
+                throw new IllegalArgumentException("Unknown SortMode [" + str + "]");
+        }
+    }
+
+    @Override
+    public String toString() {
+        return name().toLowerCase(Locale.ROOT);
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortOrder.java b/core/src/main/java/org/elasticsearch/search/sort/SortOrder.java
index 73e5ac55247..a84a456775b 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/SortOrder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/SortOrder.java
@@ -50,11 +50,8 @@ public enum SortOrder implements Writeable<SortOrder> {
             return "desc";
         }
     };
-    
-    private static final SortOrder PROTOTYPE = ASC;
 
-    @Override
-    public SortOrder readFrom(StreamInput in) throws IOException {
+    static SortOrder readFromStream(StreamInput in) throws IOException {
         int ordinal = in.readVInt();
         if (ordinal < 0 || ordinal >= values().length) {
             throw new IOException("Unknown SortOrder ordinal [" + ordinal + "]");
@@ -62,10 +59,6 @@ public enum SortOrder implements Writeable<SortOrder> {
         return values()[ordinal];
     }
 
-    public static SortOrder readOrderFrom(StreamInput in) throws IOException {
-        return PROTOTYPE.readFrom(in);
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(this.ordinal());
diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java b/core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java
deleted file mode 100644
index 5349d6fc0d6..00000000000
--- a/core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.sort;
-
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.join.BitSetProducer;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.query.support.NestedInnerQueryParseSupport;
-import org.elasticsearch.search.MultiValueMode;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.SearchParseException;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static java.util.Collections.unmodifiableMap;
-
-/**
- *
- */
-public class SortParseElement implements SearchParseElement {
-
-    public static final SortField SORT_SCORE = new SortField(null, SortField.Type.SCORE);
-    private static final SortField SORT_SCORE_REVERSE = new SortField(null, SortField.Type.SCORE, true);
-    private static final SortField SORT_DOC = new SortField(null, SortField.Type.DOC);
-    private static final SortField SORT_DOC_REVERSE = new SortField(null, SortField.Type.DOC, true);
-
-    public static final ParseField UNMAPPED_TYPE = new ParseField("unmapped_type");
-
-    public static final String SCORE_FIELD_NAME = "_score";
-    public static final String DOC_FIELD_NAME = "_doc";
-
-    private static final Map<String, SortParser> PARSERS;
-
-    static {
-        Map<String, SortParser> parsers = new HashMap<>();
-        addParser(parsers, new ScriptSortParser());
-        addParser(parsers, new GeoDistanceSortParser());
-        PARSERS = unmodifiableMap(parsers);
-    }
-
-    private static void addParser(Map<String, SortParser> parsers, SortParser parser) {
-        for (String name : parser.names()) {
-            parsers.put(name, parser);
-        }
-    }
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        XContentParser.Token token = parser.currentToken();
-        List<SortField> sortFields = new ArrayList<>(2);
-        if (token == XContentParser.Token.START_ARRAY) {
-            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                if (token == XContentParser.Token.START_OBJECT) {
-                    addCompoundSortField(parser, context, sortFields);
-                } else if (token == XContentParser.Token.VALUE_STRING) {
-                    addSortField(context, sortFields, parser.text(), false, null, null, null, null);
-                } else {
-                    throw new IllegalArgumentException("malformed sort format, within the sort array, an object, or an actual string are allowed");
-                }
-            }
-        } else if (token == XContentParser.Token.VALUE_STRING) {
-            addSortField(context, sortFields, parser.text(), false, null, null, null, null);
-        } else if (token == XContentParser.Token.START_OBJECT) {
-            addCompoundSortField(parser, context, sortFields);
-        } else {
-            throw new IllegalArgumentException("malformed sort format, either start with array, object, or an actual string");
-        }
-        if (!sortFields.isEmpty()) {
-            // optimize if we just sort on score non reversed, we don't really need sorting
-            boolean sort;
-            if (sortFields.size() > 1) {
-                sort = true;
-            } else {
-                SortField sortField = sortFields.get(0);
-                if (sortField.getType() == SortField.Type.SCORE && !sortField.getReverse()) {
-                    sort = false;
-                } else {
-                    sort = true;
-                }
-            }
-            if (sort) {
-                context.sort(new Sort(sortFields.toArray(new SortField[sortFields.size()])));
-            }
-        }
-    }
-
-    private void addCompoundSortField(XContentParser parser, SearchContext context, List<SortField> sortFields) throws Exception {
-        XContentParser.Token token;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                String fieldName = parser.currentName();
-                boolean reverse = false;
-                String missing = null;
-                String innerJsonName = null;
-                String unmappedType = null;
-                MultiValueMode sortMode = null;
-                NestedInnerQueryParseSupport nestedFilterParseHelper = null;
-                token = parser.nextToken();
-                if (token == XContentParser.Token.VALUE_STRING) {
-                    String direction = parser.text();
-                    if (direction.equals("asc")) {
-                        reverse = SCORE_FIELD_NAME.equals(fieldName);
-                    } else if (direction.equals("desc")) {
-                        reverse = !SCORE_FIELD_NAME.equals(fieldName);
-                    } else {
-                        throw new IllegalArgumentException("sort direction [" + fieldName + "] not supported");
-                    }
-                    addSortField(context, sortFields, fieldName, reverse, unmappedType, missing, sortMode, nestedFilterParseHelper);
-                } else {
-                    if (PARSERS.containsKey(fieldName)) {
-                        sortFields.add(PARSERS.get(fieldName).parse(parser, context));
-                    } else {
-                        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                            if (token == XContentParser.Token.FIELD_NAME) {
-                                innerJsonName = parser.currentName();
-                            } else if (token.isValue()) {
-                                if ("reverse".equals(innerJsonName)) {
-                                    reverse = parser.booleanValue();
-                                } else if ("order".equals(innerJsonName)) {
-                                    if ("asc".equals(parser.text())) {
-                                        reverse = SCORE_FIELD_NAME.equals(fieldName);
-                                    } else if ("desc".equals(parser.text())) {
-                                        reverse = !SCORE_FIELD_NAME.equals(fieldName);
-                                    }
-                                } else if ("missing".equals(innerJsonName)) {
-                                    missing = parser.textOrNull();
-                                } else if (context.parseFieldMatcher().match(innerJsonName, UNMAPPED_TYPE)) {
-                                    unmappedType = parser.textOrNull();
-                                } else if ("mode".equals(innerJsonName)) {
-                                    sortMode = MultiValueMode.fromString(parser.text());
-                                } else if ("nested_path".equals(innerJsonName) || "nestedPath".equals(innerJsonName)) {
-                                    if (nestedFilterParseHelper == null) {
-                                        nestedFilterParseHelper = new NestedInnerQueryParseSupport(parser, context);
-                                    }
-                                    nestedFilterParseHelper.setPath(parser.text());
-                                } else {
-                                    throw new IllegalArgumentException("sort option [" + innerJsonName + "] not supported");
-                                }
-                            } else if (token == XContentParser.Token.START_OBJECT) {
-                                if ("nested_filter".equals(innerJsonName) || "nestedFilter".equals(innerJsonName)) {
-                                    if (nestedFilterParseHelper == null) {
-                                        nestedFilterParseHelper = new NestedInnerQueryParseSupport(parser, context);
-                                    }
-                                    nestedFilterParseHelper.filter();
-                                } else {
-                                    throw new IllegalArgumentException("sort option [" + innerJsonName + "] not supported");
-                                }
-                            }
-                        }
-                        addSortField(context, sortFields, fieldName, reverse, unmappedType, missing, sortMode, nestedFilterParseHelper);
-                    }
-                }
-            }
-        }
-    }
-
-    private void addSortField(SearchContext context, List<SortField> sortFields, String fieldName, boolean reverse, String unmappedType, @Nullable final String missing, MultiValueMode sortMode, NestedInnerQueryParseSupport nestedHelper) throws IOException {
-        if (SCORE_FIELD_NAME.equals(fieldName)) {
-            if (reverse) {
-                sortFields.add(SORT_SCORE_REVERSE);
-            } else {
-                sortFields.add(SORT_SCORE);
-            }
-        } else if (DOC_FIELD_NAME.equals(fieldName)) {
-            if (reverse) {
-                sortFields.add(SORT_DOC_REVERSE);
-            } else {
-                sortFields.add(SORT_DOC);
-            }
-        } else {
-            MappedFieldType fieldType = context.smartNameFieldType(fieldName);
-            if (fieldType == null) {
-                if (unmappedType != null) {
-                    fieldType = context.mapperService().unmappedFieldType(unmappedType);
-                } else {
-                    throw new SearchParseException(context, "No mapping found for [" + fieldName + "] in order to sort on", null);
-                }
-            }
-
-            if (!fieldType.isSortable()) {
-                throw new SearchParseException(context, "Sorting not supported for field[" + fieldName + "]", null);
-            }
-
-            // Enable when we also know how to detect fields that do tokenize, but only emit one token
-            /*if (fieldMapper instanceof StringFieldMapper) {
-                StringFieldMapper stringFieldMapper = (StringFieldMapper) fieldMapper;
-                if (stringFieldMapper.fieldType().tokenized()) {
-                    // Fail early
-                    throw new SearchParseException(context, "Can't sort on tokenized string field[" + fieldName + "]");
-                }
-            }*/
-
-            // We only support AVG and SUM on number based fields
-            if (fieldType.isNumeric() == false && (sortMode == MultiValueMode.SUM || sortMode == MultiValueMode.AVG)) {
-                sortMode = null;
-            }
-            if (sortMode == null) {
-                sortMode = resolveDefaultSortMode(reverse);
-            }
-
-            final Nested nested;
-            if (nestedHelper != null && nestedHelper.getPath() != null) {
-                BitSetProducer rootDocumentsFilter = context.bitsetFilterCache().getBitSetProducer(Queries.newNonNestedFilter());
-                Query innerDocumentsFilter;
-                if (nestedHelper.filterFound()) {
-                    // TODO: use queries instead
-                    innerDocumentsFilter = nestedHelper.getInnerFilter();
-                } else {
-                    innerDocumentsFilter = nestedHelper.getNestedObjectMapper().nestedTypeFilter();
-                }
-                nested = new Nested(rootDocumentsFilter,  context.searcher().createNormalizedWeight(innerDocumentsFilter, false));
-            } else {
-                nested = null;
-            }
-
-            IndexFieldData.XFieldComparatorSource fieldComparatorSource = context.fieldData().getForField(fieldType)
-                    .comparatorSource(missing, sortMode, nested);
-            sortFields.add(new SortField(fieldType.name(), fieldComparatorSource, reverse));
-        }
-    }
-
-    private static MultiValueMode resolveDefaultSortMode(boolean reverse) {
-        return reverse ? MultiValueMode.MAX : MultiValueMode.MIN;
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/DirectSpellcheckerSettings.java b/core/src/main/java/org/elasticsearch/search/suggest/DirectSpellcheckerSettings.java
index 2b4687c8497..81c73df53fa 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/DirectSpellcheckerSettings.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/DirectSpellcheckerSettings.java
@@ -25,16 +25,29 @@ import org.apache.lucene.util.automaton.LevenshteinAutomata;
 
 public class DirectSpellcheckerSettings  {
 
-    private SuggestMode suggestMode = SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX;
-    private float accuracy = 0.5f;
-    private Suggest.Suggestion.Sort sort = Suggest.Suggestion.Sort.SCORE;
-    private StringDistance stringDistance = DirectSpellChecker.INTERNAL_LEVENSHTEIN;
-    private int maxEdits = LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE;
-    private int maxInspections = 5;
-    private float maxTermFreq = 0.01f;
-    private int prefixLength = 1;
-    private int minWordLength = 4;
-    private float minDocFreq = 0f;
+    // NB: If this changes, make sure to change the default in TermBuilderSuggester
+    public static SuggestMode DEFAULT_SUGGEST_MODE = SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX;
+    public static float DEFAULT_ACCURACY = 0.5f;
+    public static SortBy DEFAULT_SORT = SortBy.SCORE;
+    // NB: If this changes, make sure to change the default in TermBuilderSuggester
+    public static StringDistance DEFAULT_STRING_DISTANCE = DirectSpellChecker.INTERNAL_LEVENSHTEIN;
+    public static int DEFAULT_MAX_EDITS = LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE;
+    public static int DEFAULT_MAX_INSPECTIONS = 5;
+    public static float DEFAULT_MAX_TERM_FREQ = 0.01f;
+    public static int DEFAULT_PREFIX_LENGTH = 1;
+    public static int DEFAULT_MIN_WORD_LENGTH = 4;
+    public static float DEFAULT_MIN_DOC_FREQ = 0f;
+
+    private SuggestMode suggestMode = DEFAULT_SUGGEST_MODE;
+    private float accuracy = DEFAULT_ACCURACY;
+    private SortBy sort = DEFAULT_SORT;
+    private StringDistance stringDistance = DEFAULT_STRING_DISTANCE;
+    private int maxEdits = DEFAULT_MAX_EDITS;
+    private int maxInspections = DEFAULT_MAX_INSPECTIONS;
+    private float maxTermFreq = DEFAULT_MAX_TERM_FREQ;
+    private int prefixLength = DEFAULT_PREFIX_LENGTH;
+    private int minWordLength = DEFAULT_MIN_WORD_LENGTH;
+    private float minDocFreq = DEFAULT_MIN_DOC_FREQ;
 
     public SuggestMode suggestMode() {
         return suggestMode;
@@ -52,11 +65,11 @@ public class DirectSpellcheckerSettings  {
         this.accuracy = accuracy;
     }
 
-    public Suggest.Suggestion.Sort sort() {
+    public SortBy sort() {
         return sort;
     }
 
-    public void sort(Suggest.Suggestion.Sort sort) {
+    public void sort(SortBy sort) {
         this.sort = sort;
     }
 
@@ -104,8 +117,8 @@ public class DirectSpellcheckerSettings  {
         return minWordLength;
     }
 
-    public void minQueryLength(int minQueryLength) {
-        this.minWordLength = minQueryLength;
+    public void minWordLength(int minWordLength) {
+        this.minWordLength = minWordLength;
     }
 
     public float minDocFreq() {
@@ -116,4 +129,20 @@ public class DirectSpellcheckerSettings  {
         this.minDocFreq = minDocFreq;
     }
 
-}
\ No newline at end of file
+    @Override
+    public String toString() {
+        return "[" +
+                   "suggestMode=" + suggestMode +
+                   ",sort=" + sort +
+                   ",stringDistance=" + stringDistance +
+                   ",accuracy=" + accuracy +
+                   ",maxEdits=" + maxEdits +
+                   ",maxInspections=" + maxInspections +
+                   ",maxTermFreq=" + maxTermFreq +
+                   ",prefixLength=" + prefixLength +
+                   ",minWordLength=" + minWordLength +
+                   ",minDocFreq=" + minDocFreq +
+               "]";
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SortBy.java b/core/src/main/java/org/elasticsearch/search/suggest/SortBy.java
new file mode 100644
index 00000000000..14d46d134de
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/suggest/SortBy.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Objects;
+
+/**
+ * An enum representing the valid sorting options
+ */
+public enum SortBy implements Writeable<SortBy> {
+    /** Sort should first be based on score, then document frequency and then the term itself. */
+    SCORE,
+    /** Sort should first be based on document frequency, then score and then the term itself. */
+    FREQUENCY;
+
+    public static SortBy PROTOTYPE = SCORE;
+
+    @Override
+    public void writeTo(final StreamOutput out) throws IOException {
+        out.writeVInt(ordinal());
+    }
+
+    @Override
+    public SortBy readFrom(final StreamInput in) throws IOException {
+        int ordinal = in.readVInt();
+        if (ordinal < 0 || ordinal >= values().length) {
+            throw new IOException("Unknown SortBy ordinal [" + ordinal + "]");
+        }
+        return values()[ordinal];
+    }
+
+    public static SortBy resolve(final String str) {
+        Objects.requireNonNull(str, "Input string is null");
+        return valueOf(str.toUpperCase(Locale.ROOT));
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java b/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java
index 5ec92264389..f9c7092fbf1 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.search.suggest;
 
 import org.apache.lucene.util.CollectionUtil;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
@@ -47,9 +46,7 @@ import java.util.Map;
  */
 public class Suggest implements Iterable<Suggest.Suggestion<? extends Entry<? extends Option>>>, Streamable, ToXContent {
 
-    public static class Fields {
-        public static final XContentBuilderString SUGGEST = new XContentBuilderString("suggest");
-    }
+    private static final XContentBuilderString NAME = new XContentBuilderString("suggest");
 
     private static final Comparator<Option> COMPARATOR = new Comparator<Suggest.Suggestion.Entry.Option>() {
         @Override
@@ -62,26 +59,14 @@ public class Suggest implements Iterable<Suggest.Suggestion<? extends Entry<? ex
          }
     };
 
-    private final XContentBuilderString name;
-
     private List<Suggestion<? extends Entry<? extends Option>>> suggestions;
 
     private Map<String, Suggestion<? extends Entry<? extends Option>>> suggestMap;
 
     public Suggest() {
-        this.name = null;
-    }
-
-    public Suggest(XContentBuilderString name) {
-        this.name = name;
     }
 
     public Suggest(List<Suggestion<? extends Entry<? extends Option>>> suggestions) {
-        this(null, suggestions);
-    }
-
-    public Suggest(XContentBuilderString name, List<Suggestion<? extends Entry<? extends Option>>> suggestions) {
-        this.name = name;
         this.suggestions = suggestions;
     }
 
@@ -149,23 +134,24 @@ public class Suggest implements Iterable<Suggest.Suggestion<? extends Entry<? ex
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        if(name == null) {
-            for (Suggestion<?> suggestion : suggestions) {
-                suggestion.toXContent(builder, params);
-            }
-        } else {
-            builder.startObject(name);
-            for (Suggestion<?> suggestion : suggestions) {
-                suggestion.toXContent(builder, params);
-            }
-            builder.endObject();
-        }
-
+        builder.startObject(NAME);
+        toInnerXContent(builder, params);
+        builder.endObject();
         return builder;
     }
 
-    public static Suggest readSuggest(XContentBuilderString name, StreamInput in) throws IOException {
-        Suggest result = new Suggest(name);
+    /**
+     * use to write suggestion entries without <code>NAME</code> object
+     */
+    public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) throws IOException {
+        for (Suggestion<?> suggestion : suggestions) {
+            suggestion.toXContent(builder, params);
+        }
+        return builder;
+    }
+
+    public static Suggest readSuggest(StreamInput in) throws IOException {
+        Suggest result = new Suggest();
         result.readFrom(in);
         return result;
     }
@@ -198,7 +184,6 @@ public class Suggest implements Iterable<Suggest.Suggestion<? extends Entry<? ex
      */
     public static class Suggestion<T extends Suggestion.Entry> implements Iterable<T>, Streamable, ToXContent {
 
-
         public static final int TYPE = 0;
         protected String name;
         protected int size;
@@ -643,39 +628,6 @@ public class Suggest implements Iterable<Suggest.Suggestion<? extends Entry<? ex
                 }
             }
         }
-
-        public enum Sort {
-
-            /**
-             * Sort should first be based on score.
-             */
-            SCORE((byte) 0x0),
-
-            /**
-             * Sort should first be based on document frequency.
-             */
-            FREQUENCY((byte) 0x1);
-
-            private byte id;
-
-            private Sort(byte id) {
-                this.id = id;
-            }
-
-            public byte id() {
-                return id;
-            }
-
-            public static Sort fromId(byte id) {
-                if (id == 0) {
-                    return SCORE;
-                } else if (id == 1) {
-                    return FREQUENCY;
-                } else {
-                    throw new ElasticsearchException("Illegal suggest sort " + id);
-                }
-            }
-        }
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java
index 5621e03e7de..aed44c426d3 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java
@@ -19,198 +19,186 @@
 package org.elasticsearch.search.suggest;
 
 import org.elasticsearch.action.support.ToXContentToBytes;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
 
 /**
  * Defines how to perform suggesting. This builders allows a number of global options to be specified and
- * an arbitrary number of {@link org.elasticsearch.search.suggest.term.TermSuggestionBuilder} instances.
+ * an arbitrary number of {@link SuggestionBuilder} instances.
  * <p>
- * Suggesting works by suggesting terms that appear in the suggest text that are similar compared to the terms in
- * provided text. These spelling suggestions are based on several options described in this class.
+ * Suggesting works by suggesting terms/phrases that appear in the suggest text that are similar compared
+ * to the terms in provided text. These suggestions are based on several options described in this class.
  */
-public class SuggestBuilder extends ToXContentToBytes {
+public class SuggestBuilder extends ToXContentToBytes implements Writeable<SuggestBuilder> {
+
+    public static final SuggestBuilder PROTOTYPE = new SuggestBuilder();
+    protected static final ParseField GLOBAL_TEXT_FIELD = new ParseField("text");
 
-    private final String name;
     private String globalText;
+    private final Map<String, SuggestionBuilder<?>> suggestions = new HashMap<>();
 
-    private final List<SuggestionBuilder<?>> suggestions = new ArrayList<>();
-
-    public SuggestBuilder() {
-        this.name = null;
-    }
-    
-    public SuggestBuilder(String name) {
-        this.name = name;
-    }
-    
     /**
      * Sets the text to provide suggestions for. The suggest text is a required option that needs
-     * to be set either via this setter or via the {@link org.elasticsearch.search.suggest.SuggestBuilder.SuggestionBuilder#setText(String)} method.
+     * to be set either via this setter or via the {@link org.elasticsearch.search.suggest.SuggestionBuilder#text(String)} method.
      * <p>
      * The suggest text gets analyzed by the suggest analyzer or the suggest field search analyzer.
      * For each analyzed token, suggested terms are suggested if possible.
      */
-    public SuggestBuilder setText(String globalText) {
+    public SuggestBuilder setGlobalText(@Nullable String globalText) {
         this.globalText = globalText;
         return this;
     }
 
     /**
-     * Adds an {@link org.elasticsearch.search.suggest.term.TermSuggestionBuilder} instance under a user defined name.
-     * The order in which the <code>Suggestions</code> are added, is the same as in the response.
+     * Gets the global suggest text
      */
-    public SuggestBuilder addSuggestion(SuggestionBuilder<?> suggestion) {
-        suggestions.add(suggestion);
+    @Nullable
+    public String getGlobalText() {
+        return globalText;
+    }
+
+    /**
+     * Adds an {@link org.elasticsearch.search.suggest.SuggestionBuilder} instance under a user defined name.
+     * The order in which the <code>Suggestions</code> are added, is the same as in the response.
+     * @throws IllegalArgumentException if two suggestions added have the same name
+     */
+    public SuggestBuilder addSuggestion(String name, SuggestionBuilder<?> suggestion) {
+        Objects.requireNonNull(name, "every suggestion needs a name");
+        if (suggestions.get(name) == null) {
+            suggestions.put(name, suggestion);
+        } else {
+            throw new IllegalArgumentException("already added another suggestion with name [" + name + "]");
+        }
         return this;
     }
-    
+
     /**
-     * Returns all suggestions with the defined names.
+     * Get all the <code>Suggestions</code> that were added to the global {@link SuggestBuilder},
+     * together with their names
      */
-    public List<SuggestionBuilder<?>> getSuggestion() {
+    public Map<String, SuggestionBuilder<?>> getSuggestions() {
         return suggestions;
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        if(name == null) {
-            builder.startObject();
-        } else {
-            builder.startObject(name);
-        }
-        
+        builder.startObject();
         if (globalText != null) {
             builder.field("text", globalText);
         }
-        for (SuggestionBuilder<?> suggestion : suggestions) {
-            builder = suggestion.toXContent(builder, params);
+        for (Entry<String, SuggestionBuilder<?>> suggestion : suggestions.entrySet()) {
+            builder.startObject(suggestion.getKey());
+            suggestion.getValue().toXContent(builder, params);
+            builder.endObject();
         }
         builder.endObject();
         return builder;
     }
 
-    public static abstract class SuggestionBuilder<T> extends ToXContentToBytes {
+    public static SuggestBuilder fromXContent(QueryParseContext parseContext, Suggesters suggesters) throws IOException {
+        XContentParser parser = parseContext.parser();
+        ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher();
+        SuggestBuilder suggestBuilder = new SuggestBuilder();
+        String fieldName = null;
 
-        private String name;
-        private String suggester;
-        private String text;
-        private String prefix;
-        private String regex;
-        private String field;
-        private String analyzer;
-        private Integer size;
-        private Integer shardSize;
-
-        public SuggestionBuilder(String name, String suggester) {
-            this.name = name;
-            this.suggester = suggester;
+        if (parser.currentToken() == null) {
+            // when we parse from RestSuggestAction the current token is null, advance the token
+            parser.nextToken();
         }
-
-        /**
-         * Same as in {@link SuggestBuilder#setText(String)}, but in the suggestion scope.
-         */
-        @SuppressWarnings("unchecked")
-        public T text(String text) {
-            this.text = text;
-            return (T) this;
+        assert parser.currentToken() == XContentParser.Token.START_OBJECT : "current token must be a start object";
+        XContentParser.Token token;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                fieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (parseFieldMatcher.match(fieldName, GLOBAL_TEXT_FIELD)) {
+                    suggestBuilder.setGlobalText(parser.text());
+                } else {
+                    throw new IllegalArgumentException("[suggest] does not support [" + fieldName + "]");
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                String suggestionName = fieldName;
+                if (suggestionName == null) {
+                    throw new IllegalArgumentException("suggestion must have name");
+                }
+                suggestBuilder.addSuggestion(suggestionName, SuggestionBuilder.fromXContent(parseContext, suggesters));
+            } else {
+                throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + "] after [" + fieldName + "]");
+            }
         }
+        return suggestBuilder;
+    }
 
-        protected void setPrefix(String prefix) {
-            this.prefix = prefix;
+    public SuggestionSearchContext build(QueryShardContext context) throws IOException {
+        SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext();
+        for (Entry<String, SuggestionBuilder<?>> suggestion : suggestions.entrySet()) {
+            SuggestionContext suggestionContext = suggestion.getValue().build(context);
+            if (suggestionContext.getText() == null) {
+                if (globalText == null) {
+                    throw new IllegalArgumentException("The required text option is missing");
+                }
+                suggestionContext.setText(BytesRefs.toBytesRef(globalText));
+            }
+            suggestionSearchContext.addSuggestion(suggestion.getKey(), suggestionContext);
         }
+        return suggestionSearchContext;
+    }
 
-        protected void setRegex(String regex) {
-            this.regex = regex;
+    @Override
+    public SuggestBuilder readFrom(StreamInput in) throws IOException {
+        final SuggestBuilder builder = new SuggestBuilder();
+        builder.globalText = in.readOptionalString();
+        final int size = in.readVInt();
+        for (int i = 0; i < size; i++) {
+            builder.suggestions.put(in.readString(), in.readSuggestion());
         }
+        return builder;
+    }
 
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(name);
-            if (text != null) {
-                builder.field("text", text);
-            }
-            if (prefix != null) {
-                builder.field("prefix", prefix);
-            }
-            if (regex != null) {
-                builder.field("regex", regex);
-            }
-            builder.startObject(suggester);
-            if (analyzer != null) {
-                builder.field("analyzer", analyzer);
-            }
-            if (field != null) {
-                builder.field("field", field);
-            }
-            if (size != null) {
-                builder.field("size", size);
-            }
-            if (shardSize != null) {
-                builder.field("shard_size", shardSize);
-            }
-
-            builder = innerToXContent(builder, params);
-            builder.endObject();
-            builder.endObject();
-            return builder;
-        }
-
-        protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException;
-
-        /**
-         * Sets from what field to fetch the candidate suggestions from. This is an
-         * required option and needs to be set via this setter or
-         * {@link org.elasticsearch.search.suggest.term.TermSuggestionBuilder#field(String)}
-         * method
-         */
-        @SuppressWarnings("unchecked")
-        public T field(String field) {
-            this.field = field;
-            return (T)this;
-        }
-
-        /**
-         * Sets the analyzer to analyse to suggest text with. Defaults to the search
-         * analyzer of the suggest field.
-         */
-        @SuppressWarnings("unchecked")
-        public T analyzer(String analyzer) {
-            this.analyzer = analyzer;
-            return (T)this;
-        }
-
-        /**
-         * Sets the maximum suggestions to be returned per suggest text term.
-         */
-        @SuppressWarnings("unchecked")
-        public T size(int size) {
-            if (size <= 0) {
-                throw new IllegalArgumentException("Size must be positive");
-            }
-            this.size = size;
-            return (T)this;
-        }
-
-        /**
-         * Sets the maximum number of suggested term to be retrieved from each
-         * individual shard. During the reduce phase the only the top N suggestions
-         * are returned based on the <code>size</code> option. Defaults to the
-         * <code>size</code> option.
-         * <p>
-         * Setting this to a value higher than the `size` can be useful in order to
-         * get a more accurate document frequency for suggested terms. Due to the
-         * fact that terms are partitioned amongst shards, the shard level document
-         * frequencies of suggestions may not be precise. Increasing this will make
-         * these document frequencies more precise.
-         */
-        @SuppressWarnings("unchecked")
-        public T shardSize(Integer shardSize) {
-            this.shardSize = shardSize;
-            return (T)this;
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalString(globalText);
+        final int size = suggestions.size();
+        out.writeVInt(size);
+        for (Entry<String, SuggestionBuilder<?>> suggestion : suggestions.entrySet()) {
+            out.writeString(suggestion.getKey());
+            out.writeSuggestion(suggestion.getValue());
         }
     }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+        @SuppressWarnings("unchecked")
+        SuggestBuilder o = (SuggestBuilder)other;
+        return Objects.equals(globalText, o.globalText) &&
+               Objects.equals(suggestions, o.suggestions);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(globalText, suggestions);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilders.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilders.java
index 66b917394ff..6050d4ffb4e 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilders.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilders.java
@@ -29,35 +29,32 @@ import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 public abstract class SuggestBuilders {
 
     /**
-     * Creates a term suggestion lookup query with the provided <code>name</code>
+     * Creates a term suggestion lookup query with the provided <code>field</code>
      *
-     * @param name The suggestion name
      * @return a {@link org.elasticsearch.search.suggest.term.TermSuggestionBuilder}
      * instance
      */
-    public static TermSuggestionBuilder termSuggestion(String name) {
-        return new TermSuggestionBuilder(name);
+    public static TermSuggestionBuilder termSuggestion(String fieldname) {
+        return new TermSuggestionBuilder(fieldname);
     }
 
     /**
-     * Creates a phrase suggestion lookup query with the provided <code>name</code>
+     * Creates a phrase suggestion lookup query with the provided <code>field</code>
      *
-     * @param name The suggestion name
      * @return a {@link org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder}
      * instance
      */
-    public static PhraseSuggestionBuilder phraseSuggestion(String name) {
-        return new PhraseSuggestionBuilder(name);
+    public static PhraseSuggestionBuilder phraseSuggestion(String fieldname) {
+        return new PhraseSuggestionBuilder(fieldname);
     }
 
     /**
-     * Creates a completion suggestion lookup query with the provided <code>name</code>
+     * Creates a completion suggestion lookup query with the provided <code>field</code>
      *
-     * @param name The suggestion name
      * @return a {@link org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder}
      * instance
      */
-    public static CompletionSuggestionBuilder completionSuggestion(String name) {
-        return new CompletionSuggestionBuilder(name);
+    public static CompletionSuggestionBuilder completionSuggestion(String fieldname) {
+        return new CompletionSuggestionBuilder(fieldname);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java
deleted file mode 100644
index a8a4e9ec26b..00000000000
--- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.suggest;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.IndexFieldDataService;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- *
- */
-public final class SuggestParseElement implements SearchParseElement {
-    private Suggesters suggesters;
-
-    @Inject
-    public SuggestParseElement(Suggesters suggesters) {
-        this.suggesters = suggesters;
-    }
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        SuggestionSearchContext suggestionSearchContext = parseInternal(parser, context.mapperService(), context.fieldData(),
-                context.shardTarget().index(), context.shardTarget().shardId());
-        context.suggest(suggestionSearchContext);
-    }
-
-    public SuggestionSearchContext parseInternal(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService,
-                                                 String index, int shardId) throws IOException {
-        SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext();
-
-        BytesRef globalText = null;
-        String fieldName = null;
-        Map<String, SuggestionContext> suggestionContexts = new HashMap<>();
-
-        XContentParser.Token token;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                fieldName = parser.currentName();
-            } else if (token.isValue()) {
-                if ("text".equals(fieldName)) {
-                    globalText = parser.utf8Bytes();
-                } else {
-                    throw new IllegalArgumentException("[suggest] does not support [" + fieldName + "]");
-                }
-            } else if (token == XContentParser.Token.START_OBJECT) {
-                String suggestionName = fieldName;
-                BytesRef suggestText = null;
-                BytesRef prefix = null;
-                BytesRef regex = null;
-                SuggestionContext suggestionContext = null;
-
-                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                    if (token == XContentParser.Token.FIELD_NAME) {
-                        fieldName = parser.currentName();
-                    } else if (token.isValue()) {
-                        if ("text".equals(fieldName)) {
-                            suggestText = parser.utf8Bytes();
-                        } else if ("prefix".equals(fieldName)) {
-                            prefix = parser.utf8Bytes();
-                        } else if ("regex".equals(fieldName)) {
-                            regex = parser.utf8Bytes();
-                        } else {
-                            throw new IllegalArgumentException("[suggest] does not support [" + fieldName + "]");
-                        }
-                    } else if (token == XContentParser.Token.START_OBJECT) {
-                        if (suggestionName == null) {
-                            throw new IllegalArgumentException("Suggestion must have name");
-                        }
-                        if (suggesters.get(fieldName) == null) {
-                            throw new IllegalArgumentException("Suggester[" + fieldName + "] not supported");
-                        }
-                        final SuggestContextParser contextParser = suggesters.get(fieldName).getContextParser();
-                        suggestionContext = contextParser.parse(parser, mapperService, fieldDataService);
-                    }
-                }
-                if (suggestionContext != null) {
-                    if (suggestText != null && prefix == null) {
-                        suggestionContext.setPrefix(suggestText);
-                        suggestionContext.setText(suggestText);
-                    } else if (suggestText == null && prefix != null) {
-                        suggestionContext.setPrefix(prefix);
-                        suggestionContext.setText(prefix);
-                    } else if (regex != null) {
-                        suggestionContext.setRegex(regex);
-                        suggestionContext.setText(regex);
-                    }
-                    suggestionContexts.put(suggestionName, suggestionContext);
-                }
-            }
-        }
-
-        for (Map.Entry<String, SuggestionContext> entry : suggestionContexts.entrySet()) {
-            String suggestionName = entry.getKey();
-            SuggestionContext suggestionContext = entry.getValue();
-
-            suggestionContext.setShard(shardId);
-            suggestionContext.setIndex(index);
-            SuggestUtils.verifySuggestion(mapperService, globalText, suggestionContext);
-            suggestionSearchContext.addSuggestion(suggestionName, suggestionContext);
-        }
-
-        return suggestionSearchContext;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java
index 541efa7ef43..520322587bb 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.search.suggest;
 
-import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.util.CharsRefBuilder;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.component.AbstractComponent;
@@ -37,28 +36,22 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
-import static java.util.Collections.singletonMap;
+import static java.util.Collections.emptyMap;
 
 /**
  */
 public class SuggestPhase extends AbstractComponent implements SearchPhase {
-    private final Map<String, SearchParseElement> parseElements;
-    private final SuggestParseElement parseElement;
 
     @Inject
-    public SuggestPhase(Settings settings, SuggestParseElement suggestParseElement) {
+    public SuggestPhase(Settings settings) {
         super(settings);
-        this.parseElement = suggestParseElement;
-        parseElements = singletonMap("suggest", parseElement);
     }
 
     @Override
     public Map<String, ? extends SearchParseElement> parseElements() {
-        return parseElements;
-    }
-
-    public SuggestParseElement parseElement() {
-        return parseElement;
+        // this is used to parse SearchSourceBuilder.ext() bytes
+        // we don't allow any suggestion parsing for the extension
+        return emptyMap();
     }
 
     @Override
@@ -71,10 +64,6 @@ public class SuggestPhase extends AbstractComponent implements SearchPhase {
         if (suggest == null) {
             return;
         }
-        context.queryResult().suggest(execute(suggest, context.searcher()));
-    }
-
-    public Suggest execute(SuggestionSearchContext suggest, IndexSearcher searcher) {
         try {
             CharsRefBuilder spare = new CharsRefBuilder();
             final List<Suggestion<? extends Entry<? extends Option>>> suggestions = new ArrayList<>(suggest.suggestions().size());
@@ -82,14 +71,14 @@ public class SuggestPhase extends AbstractComponent implements SearchPhase {
             for (Map.Entry<String, SuggestionSearchContext.SuggestionContext> entry : suggest.suggestions().entrySet()) {
                 SuggestionSearchContext.SuggestionContext suggestion = entry.getValue();
                 Suggester<SuggestionContext> suggester = suggestion.getSuggester();
-                Suggestion<? extends Entry<? extends Option>> result = suggester.execute(entry.getKey(), suggestion, searcher, spare);
+                Suggestion<? extends Entry<? extends Option>> result =
+                    suggester.execute(entry.getKey(), suggestion, context.searcher(), spare);
                 if (result != null) {
                     assert entry.getKey().equals(result.name);
                     suggestions.add(result);
                 }
             }
-
-            return new Suggest(Suggest.Fields.SUGGEST, suggestions);
+            context.queryResult().suggest(new Suggest(suggestions));
         } catch (IOException e) {
             throw new ElasticsearchException("I/O exception during suggest phase", e);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java
index 2509f792ecc..5cf599beda4 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java
@@ -170,17 +170,8 @@ public final class SuggestUtils {
         }
     }
 
-    public static Suggest.Suggestion.Sort resolveSort(String sortVal) {
-        if ("score".equals(sortVal)) {
-            return Suggest.Suggestion.Sort.SCORE;
-        } else if ("frequency".equals(sortVal)) {
-            return Suggest.Suggestion.Sort.FREQUENCY;
-        } else {
-            throw new IllegalArgumentException("Illegal suggest sort " + sortVal);
-        }
-    }
-
     public static StringDistance resolveDistance(String distanceVal) {
+        distanceVal = distanceVal.toLowerCase(Locale.US);
         if ("internal".equals(distanceVal)) {
             return DirectSpellChecker.INTERNAL_LEVENSHTEIN;
         } else if ("damerau_levenshtein".equals(distanceVal) || "damerauLevenshtein".equals(distanceVal)) {
@@ -224,24 +215,24 @@ public final class SuggestUtils {
             } else if (parseFieldMatcher.match(fieldName, Fields.SUGGEST_MODE)) {
                 suggestion.suggestMode(SuggestUtils.resolveSuggestMode(parser.text()));
             } else if (parseFieldMatcher.match(fieldName, Fields.SORT)) {
-                suggestion.sort(SuggestUtils.resolveSort(parser.text()));
+                suggestion.sort(SortBy.resolve(parser.text()));
             } else if (parseFieldMatcher.match(fieldName, Fields.STRING_DISTANCE)) {
-            suggestion.stringDistance(SuggestUtils.resolveDistance(parser.text()));
+                suggestion.stringDistance(SuggestUtils.resolveDistance(parser.text()));
             } else if (parseFieldMatcher.match(fieldName, Fields.MAX_EDITS)) {
-            suggestion.maxEdits(parser.intValue());
+                suggestion.maxEdits(parser.intValue());
                 if (suggestion.maxEdits() < 1 || suggestion.maxEdits() > LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE) {
                     throw new IllegalArgumentException("Illegal max_edits value " + suggestion.maxEdits());
                 }
             } else if (parseFieldMatcher.match(fieldName, Fields.MAX_INSPECTIONS)) {
-            suggestion.maxInspections(parser.intValue());
+                suggestion.maxInspections(parser.intValue());
             } else if (parseFieldMatcher.match(fieldName, Fields.MAX_TERM_FREQ)) {
-            suggestion.maxTermFreq(parser.floatValue());
+                suggestion.maxTermFreq(parser.floatValue());
             } else if (parseFieldMatcher.match(fieldName, Fields.PREFIX_LENGTH)) {
-            suggestion.prefixLength(parser.intValue());
+                suggestion.prefixLength(parser.intValue());
             } else if (parseFieldMatcher.match(fieldName, Fields.MIN_WORD_LENGTH)) {
-            suggestion.minQueryLength(parser.intValue());
+                suggestion.minWordLength(parser.intValue());
             } else if (parseFieldMatcher.match(fieldName, Fields.MIN_DOC_FREQ)) {
-            suggestion.minDocFreq(parser.floatValue());
+                suggestion.minDocFreq(parser.floatValue());
             } else {
                 return false;
             }
@@ -268,10 +259,8 @@ public final class SuggestUtils {
            return false;
         }
         return true;
-
     }
 
-
     public static void verifySuggestion(MapperService mapperService, BytesRef globalText, SuggestionContext suggestion) {
         // Verify options and set defaults
         if (suggestion.getField() == null) {
@@ -291,7 +280,6 @@ public final class SuggestUtils {
         }
     }
 
-
     public static ShingleTokenFilterFactory.Factory getShingleFilterFactory(Analyzer analyzer) {
         if (analyzer instanceof NamedAnalyzer) {
             analyzer = ((NamedAnalyzer)analyzer).analyzer();
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/Suggester.java b/core/src/main/java/org/elasticsearch/search/suggest/Suggester.java
index 7b3f7bdb89f..5772f2b55d3 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/Suggester.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/Suggester.java
@@ -29,7 +29,10 @@ public abstract class Suggester<T extends SuggestionSearchContext.SuggestionCont
     protected abstract Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
         innerExecute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException;
 
-    public abstract SuggestContextParser getContextParser();
+    /**
+     * link the suggester to its corresponding {@link SuggestionBuilder}
+     */
+    public abstract SuggestionBuilder<? extends SuggestionBuilder> getBuilderPrototype();
 
     public Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
         execute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java b/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java
index af54e5dfd86..644d9239b8c 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java
@@ -20,8 +20,6 @@ package org.elasticsearch.search.suggest;
 
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.ExtensionPoint;
-import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.suggest.completion.CompletionSuggester;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggester;
 import org.elasticsearch.search.suggest.term.TermSuggester;
@@ -42,26 +40,30 @@ public final class Suggesters extends ExtensionPoint.ClassMap<Suggester> {
         this(Collections.emptyMap());
     }
 
-    public Suggesters(Map<String, Suggester> suggesters) {
-        super("suggester", Suggester.class, new HashSet<>(Arrays.asList("phrase", "term", "completion")), Suggesters.class, SuggestParseElement.class, SuggestPhase.class);
-        this.parsers = Collections.unmodifiableMap(suggesters);
-    }
-
     @Inject
-    public Suggesters(Map<String, Suggester> suggesters, ScriptService scriptService, IndicesService indexServices) {
-        this(addBuildIns(suggesters, scriptService, indexServices));
+    public Suggesters(Map<String, Suggester> suggesters) {
+        super("suggester", Suggester.class, new HashSet<>(Arrays.asList("phrase", "term", "completion")), Suggesters.class, SuggestPhase.class);
+        this.parsers = Collections.unmodifiableMap(addBuildIns(suggesters));
     }
 
-    private static Map<String, Suggester> addBuildIns(Map<String, Suggester> suggesters, ScriptService scriptService, IndicesService indexServices) {
+    private static Map<String, Suggester> addBuildIns(Map<String, Suggester> suggesters) {
         final Map<String, Suggester> map = new HashMap<>();
-        map.put("phrase", new PhraseSuggester(scriptService, indexServices));
-        map.put("term", new TermSuggester());
-        map.put("completion", new CompletionSuggester());
+        map.put("phrase", PhraseSuggester.PROTOTYPE);
+        map.put("term", TermSuggester.PROTOTYPE);
+        map.put("completion", CompletionSuggester.PROTOTYPE);
         map.putAll(suggesters);
         return map;
     }
 
-    public Suggester get(String type) {
-        return parsers.get(type);
+    public SuggestionBuilder<? extends SuggestionBuilder> getSuggestionPrototype(String suggesterName) {
+        Suggester<?> suggester = parsers.get(suggesterName);
+        if (suggester == null) {
+            throw new IllegalArgumentException("suggester with name [" + suggesterName + "] not supported");
+        }
+        SuggestionBuilder<?> suggestParser = suggester.getBuilderPrototype();
+        if (suggestParser == null) {
+            throw new IllegalArgumentException("suggester with name [" + suggesterName + "] not supported");
+        }
+        return suggestParser;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java
new file mode 100644
index 00000000000..29f50649cb1
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java
@@ -0,0 +1,412 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.support.ToXContentToBytes;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Base class for the different suggestion implementations.
+ */
+public abstract class SuggestionBuilder<T extends SuggestionBuilder<T>> extends ToXContentToBytes implements NamedWriteable<T> {
+
+    protected final String field;
+    protected String text;
+    protected String prefix;
+    protected String regex;
+    protected String analyzer;
+    protected Integer size;
+    protected Integer shardSize;
+
+    protected static final ParseField TEXT_FIELD = new ParseField("text");
+    protected static final ParseField PREFIX_FIELD = new ParseField("prefix");
+    protected static final ParseField REGEX_FIELD = new ParseField("regex");
+    protected static final ParseField FIELDNAME_FIELD = new ParseField("field");
+    protected static final ParseField ANALYZER_FIELD = new ParseField("analyzer");
+    protected static final ParseField SIZE_FIELD = new ParseField("size");
+    protected static final ParseField SHARDSIZE_FIELD = new ParseField("shard_size");
+
+    /**
+     * Creates a new suggestion.
+     * @param field field to execute suggestions on
+     */
+    protected SuggestionBuilder(String field) {
+        Objects.requireNonNull(field, "suggestion requires a field name");
+        if (field.isEmpty()) {
+            throw new IllegalArgumentException("suggestion field name is empty");
+        }
+        this.field = field;
+    }
+
+    /**
+     * internal copy constructor that copies over all class fields from second SuggestionBuilder except field name.
+     */
+    protected SuggestionBuilder(String field, SuggestionBuilder<?> in) {
+        this(field);
+        text = in.text;
+        prefix = in.prefix;
+        regex = in.regex;
+        analyzer = in.analyzer;
+        size = in.size;
+        shardSize = in.shardSize;
+    }
+
+    /**
+     * Same as in {@link SuggestBuilder#setGlobalText(String)}, but in the suggestion scope.
+     */
+    @SuppressWarnings("unchecked")
+    public T text(String text) {
+        this.text = text;
+        return (T) this;
+    }
+
+    /**
+     * get the text for this suggestion
+     */
+    public String text() {
+        return this.text;
+    }
+
+    @SuppressWarnings("unchecked")
+    protected T prefix(String prefix) {
+        this.prefix = prefix;
+        return (T) this;
+    }
+
+    /**
+     * get the prefix for this suggestion
+     */
+    public String prefix() {
+        return this.prefix;
+    }
+
+    @SuppressWarnings("unchecked")
+    protected T regex(String regex) {
+        this.regex = regex;
+        return (T) this;
+    }
+
+    /**
+     * get the regex for this suggestion
+     */
+    public String regex() {
+        return this.regex;
+    }
+
+    /**
+     * get the {@link #field()} parameter
+     */
+    public String field() {
+        return this.field;
+    }
+
+    /**
+     * Sets the analyzer to analyse to suggest text with. Defaults to the search
+     * analyzer of the suggest field.
+     */
+    @SuppressWarnings("unchecked")
+    public T analyzer(String analyzer) {
+        this.analyzer = analyzer;
+        return (T)this;
+    }
+
+    /**
+     * get the {@link #analyzer()} parameter
+     */
+    public String analyzer() {
+        return this.analyzer;
+    }
+
+    /**
+     * Sets the maximum suggestions to be returned per suggest text term.
+     */
+    @SuppressWarnings("unchecked")
+    public T size(int size) {
+        if (size <= 0) {
+            throw new IllegalArgumentException("size must be positive");
+        }
+        this.size = size;
+        return (T)this;
+    }
+
+    /**
+     * get the {@link #size()} parameter
+     */
+    public Integer size() {
+        return this.size;
+    }
+
+    /**
+     * Sets the maximum number of suggested term to be retrieved from each
+     * individual shard. During the reduce phase the only the top N suggestions
+     * are returned based on the <code>size</code> option. Defaults to the
+     * <code>size</code> option.
+     * <p>
+     * Setting this to a value higher than the `size` can be useful in order to
+     * get a more accurate document frequency for suggested terms. Due to the
+     * fact that terms are partitioned amongst shards, the shard level document
+     * frequencies of suggestions may not be precise. Increasing this will make
+     * these document frequencies more precise.
+     */
+    @SuppressWarnings("unchecked")
+    public T shardSize(Integer shardSize) {
+        this.shardSize = shardSize;
+        return (T)this;
+    }
+
+    /**
+     * get the {@link #shardSize()} parameter
+     */
+    public Integer shardSize() {
+        return this.shardSize;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        if (text != null) {
+            builder.field(TEXT_FIELD.getPreferredName(), text);
+        }
+        if (prefix != null) {
+            builder.field(PREFIX_FIELD.getPreferredName(), prefix);
+        }
+        if (regex != null) {
+            builder.field(REGEX_FIELD.getPreferredName(), regex);
+        }
+        builder.startObject(getSuggesterName());
+        if (analyzer != null) {
+            builder.field(ANALYZER_FIELD.getPreferredName(), analyzer);
+        }
+        builder.field(FIELDNAME_FIELD.getPreferredName(), field);
+        if (size != null) {
+            builder.field(SIZE_FIELD.getPreferredName(), size);
+        }
+        if (shardSize != null) {
+            builder.field(SHARDSIZE_FIELD.getPreferredName(), shardSize);
+        }
+
+        builder = innerToXContent(builder, params);
+        builder.endObject();
+        return builder;
+    }
+
+    protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException;
+
+    static SuggestionBuilder<?> fromXContent(QueryParseContext parseContext, Suggesters suggesters)
+            throws IOException {
+        XContentParser parser = parseContext.parser();
+        ParseFieldMatcher parsefieldMatcher = parseContext.parseFieldMatcher();
+        XContentParser.Token token;
+        String currentFieldName = null;
+        String suggestText = null;
+        String prefix = null;
+        String regex = null;
+        SuggestionBuilder<?> suggestionBuilder = null;
+
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (parsefieldMatcher.match(currentFieldName, TEXT_FIELD)) {
+                    suggestText = parser.text();
+                } else if (parsefieldMatcher.match(currentFieldName, PREFIX_FIELD)) {
+                    prefix = parser.text();
+                } else if (parsefieldMatcher.match(currentFieldName, REGEX_FIELD)) {
+                    regex = parser.text();
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "suggestion does not support [" + currentFieldName + "]");
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                SuggestionBuilder<?> suggestParser = suggesters.getSuggestionPrototype(currentFieldName);
+                if (suggestParser == null) {
+                    throw new ParsingException(parser.getTokenLocation(), "suggestion [" + currentFieldName + "] not supported");
+                }
+                suggestionBuilder = suggestParser.innerFromXContent(parseContext);
+            }
+        }
+        if (suggestionBuilder == null) {
+            throw new ElasticsearchParseException("missing suggestion object");
+        }
+        if (suggestText != null) {
+            suggestionBuilder.text(suggestText);
+        }
+        if (prefix != null) {
+            suggestionBuilder.prefix(prefix);
+        }
+        if (regex != null) {
+            suggestionBuilder.regex(regex);
+        }
+        return suggestionBuilder;
+    }
+
+    protected abstract SuggestionBuilder<T> innerFromXContent(QueryParseContext parseContext) throws IOException;
+
+    protected abstract SuggestionContext build(QueryShardContext context) throws IOException;
+
+    /**
+     * Transfers the text, prefix, regex, analyzer, field, size and shard size settings from the
+     * original {@link SuggestionBuilder} to the target {@link SuggestionContext}
+     */
+    protected void populateCommonFields(MapperService mapperService,
+            SuggestionSearchContext.SuggestionContext suggestionContext) throws IOException {
+
+        Objects.requireNonNull(field, "field must not be null");
+
+        MappedFieldType fieldType = mapperService.fullName(field);
+        if (fieldType == null) {
+            throw new IllegalArgumentException("no mapping found for field [" + field + "]");
+        } else if (analyzer == null) {
+            // no analyzer name passed in, so try the field's analyzer, or the default analyzer
+            if (fieldType.searchAnalyzer() == null) {
+                suggestionContext.setAnalyzer(mapperService.searchAnalyzer());
+            } else {
+                suggestionContext.setAnalyzer(fieldType.searchAnalyzer());
+            }
+        } else {
+            Analyzer luceneAnalyzer = mapperService.analysisService().analyzer(analyzer);
+            if (luceneAnalyzer == null) {
+                throw new IllegalArgumentException("analyzer [" + analyzer + "] doesn't exists");
+            }
+            suggestionContext.setAnalyzer(luceneAnalyzer);
+        }
+
+        suggestionContext.setField(field);
+
+        if (size != null) {
+            suggestionContext.setSize(size);
+        }
+
+        if (shardSize != null) {
+            suggestionContext.setShardSize(shardSize);
+        } else {
+            // if no shard size is set in builder, use size (or at least 5)
+            suggestionContext.setShardSize(Math.max(suggestionContext.getSize(), 5));
+        }
+
+        if (text != null) {
+            suggestionContext.setText(BytesRefs.toBytesRef(text));
+        }
+        if (prefix != null) {
+            suggestionContext.setPrefix(BytesRefs.toBytesRef(prefix));
+        }
+        if (regex != null) {
+            suggestionContext.setRegex(BytesRefs.toBytesRef(regex));
+        }
+        if (text != null && prefix == null) {
+            suggestionContext.setPrefix(BytesRefs.toBytesRef(text));
+        } else if (text == null && prefix != null) {
+            suggestionContext.setText(BytesRefs.toBytesRef(prefix));
+        } else if (text == null && regex != null) {
+            suggestionContext.setText(BytesRefs.toBytesRef(regex));
+        }
+    }
+
+    private String getSuggesterName() {
+        //default impl returns the same as writeable name, but we keep the distinction between the two just to make sure
+        return getWriteableName();
+    }
+
+    @Override
+    public final T readFrom(StreamInput in) throws IOException {
+        String field = in.readString();
+        T suggestionBuilder = doReadFrom(in, field);
+        suggestionBuilder.text = in.readOptionalString();
+        suggestionBuilder.prefix = in.readOptionalString();
+        suggestionBuilder.regex = in.readOptionalString();
+        suggestionBuilder.analyzer = in.readOptionalString();
+        suggestionBuilder.size = in.readOptionalVInt();
+        suggestionBuilder.shardSize = in.readOptionalVInt();
+        return suggestionBuilder;
+    }
+
+    /**
+     * Subclass should return a new instance, reading itself from the input string
+     * @param in the input string to read from
+     * @param field the field needed for ctor or concrete suggestion
+     */
+    protected abstract T doReadFrom(StreamInput in, String field) throws IOException;
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        out.writeString(field);
+        doWriteTo(out);
+        out.writeOptionalString(text);
+        out.writeOptionalString(prefix);
+        out.writeOptionalString(regex);
+        out.writeOptionalString(analyzer);
+        out.writeOptionalVInt(size);
+        out.writeOptionalVInt(shardSize);
+    }
+
+    protected abstract void doWriteTo(StreamOutput out) throws IOException;
+
+    @Override
+    public final boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        @SuppressWarnings("unchecked")
+        T other = (T) obj;
+        return Objects.equals(text, other.text()) &&
+               Objects.equals(prefix, other.prefix()) &&
+               Objects.equals(regex, other.regex()) &&
+               Objects.equals(field, other.field()) &&
+               Objects.equals(analyzer, other.analyzer()) &&
+               Objects.equals(size, other.size()) &&
+               Objects.equals(shardSize, other.shardSize()) &&
+               doEquals(other);
+    }
+
+    /**
+     * Indicates whether some other {@link SuggestionBuilder} of the same type is "equal to" this one.
+     */
+    protected abstract boolean doEquals(T other);
+
+    @Override
+    public final int hashCode() {
+        return Objects.hash(text, prefix, regex, field, analyzer, size, shardSize, doHashCode());
+    }
+
+    /**
+     * HashCode for the subclass of {@link SuggestionBuilder} to implement.
+     */
+    protected abstract int doHashCode();
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java
index 1d3339e0578..ece5a9c58bf 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java
@@ -20,6 +20,7 @@ package org.elasticsearch.search.suggest;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.query.QueryShardContext;
 
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -36,20 +37,24 @@ public class SuggestionSearchContext {
     public Map<String, SuggestionContext> suggestions() {
         return suggestions;
     }
-    
-    public static class SuggestionContext {
-        
+
+    public abstract static class SuggestionContext {
+
         private BytesRef text;
         private BytesRef prefix;
         private BytesRef regex;
-        private final Suggester suggester;
         private String field;
         private Analyzer analyzer;
         private int size = 5;
         private int shardSize = -1;
-        private int shardId;
-        private String index;
-        
+        private QueryShardContext shardContext;
+        private Suggester<?> suggester;
+
+        protected SuggestionContext(Suggester<?> suggester, QueryShardContext shardContext) {
+            this.suggester = suggester;
+            this.shardContext = shardContext;
+        }
+
         public BytesRef getText() {
             return text;
         }
@@ -74,12 +79,8 @@ public class SuggestionSearchContext {
             this.regex = regex;
         }
 
-        public SuggestionContext(Suggester suggester) {
-            this.suggester = suggester;
-        }
-
         public Suggester<SuggestionContext> getSuggester() {
-            return this.suggester;
+            return ((Suggester<SuggestionContext>) suggester);
         }
 
         public Analyzer getAnalyzer() {
@@ -119,21 +120,24 @@ public class SuggestionSearchContext {
             }
             this.shardSize = shardSize;
         }
-        
-        public void setShard(int shardId) {
-            this.shardId = shardId;
+
+        public QueryShardContext getShardContext() {
+            return this.shardContext;
         }
 
-        public void setIndex(String index) {
-            this.index = index;
-        }
-        
-        public String getIndex() {
-            return index;
-        }
-        
-        public int getShard() {
-            return shardId;
+        @Override
+        public String toString() {
+            return "[" +
+                       "text=" + text +
+                       ",field=" + field +
+                       ",prefix=" + prefix +
+                       ",regex=" + regex +
+                       ",size=" + size +
+                       ",shardSize=" + shardSize +
+                       ",suggester=" + suggester +
+                       ",analyzer=" + analyzer +
+                       ",shardContext=" + shardContext +
+                   "]";
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java
deleted file mode 100644
index 702b03f359e..00000000000
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.suggest.completion;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.unit.Fuzziness;
-import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.IndexFieldDataService;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
-import org.elasticsearch.index.query.RegexpFlag;
-import org.elasticsearch.search.suggest.SuggestContextParser;
-import org.elasticsearch.search.suggest.SuggestUtils.Fields;
-import org.elasticsearch.search.suggest.SuggestionSearchContext;
-import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder.FuzzyOptionsBuilder;
-import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder.RegexOptionsBuilder;
-import org.elasticsearch.search.suggest.completion.context.ContextMapping;
-import org.elasticsearch.search.suggest.completion.context.ContextMappings;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Parses query options for {@link CompletionSuggester}
- *
- * Acceptable input:
- * {
- *     "field" : STRING
- *     "size" : INT
- *     "fuzzy" : BOOLEAN | FUZZY_OBJECT
- *     "contexts" : QUERY_CONTEXTS
- *     "regex" : REGEX_OBJECT
- * }
- *
- * FUZZY_OBJECT : {
- *     "edit_distance" : STRING | INT
- *     "transpositions" : BOOLEAN
- *     "min_length" : INT
- *     "prefix_length" : INT
- *     "unicode_aware" : BOOLEAN
- *     "max_determinized_states" : INT
- * }
- *
- * REGEX_OBJECT: {
- *     "flags" : REGEX_FLAGS
- *     "max_determinized_states" : INT
- * }
- *
- * see {@link RegexpFlag} for REGEX_FLAGS
- */
-public class CompletionSuggestParser implements SuggestContextParser {
-
-    private static ObjectParser<CompletionSuggestionContext, ContextAndSuggest> TLP_PARSER = new ObjectParser<>(CompletionSuggestionBuilder.SUGGESTION_NAME, null);
-    private static ObjectParser<CompletionSuggestionBuilder.RegexOptionsBuilder, ContextAndSuggest> REGEXP_PARSER = new ObjectParser<>(RegexOptionsBuilder.REGEX_OPTIONS.getPreferredName(), CompletionSuggestionBuilder.RegexOptionsBuilder::new);
-    private static ObjectParser<CompletionSuggestionBuilder.FuzzyOptionsBuilder, ContextAndSuggest> FUZZY_PARSER = new ObjectParser<>(FuzzyOptionsBuilder.FUZZY_OPTIONS.getPreferredName(), CompletionSuggestionBuilder.FuzzyOptionsBuilder::new);
-    static {
-        FUZZY_PARSER.declareInt(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setFuzzyMinLength, FuzzyOptionsBuilder.MIN_LENGTH_FIELD);
-        FUZZY_PARSER.declareInt(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setMaxDeterminizedStates, FuzzyOptionsBuilder.MAX_DETERMINIZED_STATES_FIELD);
-        FUZZY_PARSER.declareBoolean(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setUnicodeAware, FuzzyOptionsBuilder.UNICODE_AWARE_FIELD);
-        FUZZY_PARSER.declareInt(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setFuzzyPrefixLength, FuzzyOptionsBuilder.PREFIX_LENGTH_FIELD);
-        FUZZY_PARSER.declareBoolean(CompletionSuggestionBuilder.FuzzyOptionsBuilder::setTranspositions, FuzzyOptionsBuilder.TRANSPOSITION_FIELD);
-        FUZZY_PARSER.declareValue((a, b) -> {
-            try {
-                a.setFuzziness(Fuzziness.parse(b).asDistance());
-            } catch (IOException e) {
-                throw new ElasticsearchException(e);
-            }
-        }, Fuzziness.FIELD);
-        REGEXP_PARSER.declareInt(CompletionSuggestionBuilder.RegexOptionsBuilder::setMaxDeterminizedStates, RegexOptionsBuilder.MAX_DETERMINIZED_STATES);
-        REGEXP_PARSER.declareStringOrNull(CompletionSuggestionBuilder.RegexOptionsBuilder::setFlags, RegexOptionsBuilder.FLAGS_VALUE);
-
-        TLP_PARSER.declareStringArray(CompletionSuggestionContext::setPayloadFields, CompletionSuggestionBuilder.PAYLOAD_FIELD);
-        TLP_PARSER.declareObjectOrDefault(CompletionSuggestionContext::setFuzzyOptionsBuilder, FUZZY_PARSER, CompletionSuggestionBuilder.FuzzyOptionsBuilder::new, FuzzyOptionsBuilder.FUZZY_OPTIONS);
-        TLP_PARSER.declareObject(CompletionSuggestionContext::setRegexOptionsBuilder, REGEXP_PARSER, RegexOptionsBuilder.REGEX_OPTIONS);
-        TLP_PARSER.declareString(SuggestionSearchContext.SuggestionContext::setField, Fields.FIELD);
-        TLP_PARSER.declareField((p, v, c) -> {
-            String analyzerName = p.text();
-            Analyzer analyzer = c.mapperService.analysisService().analyzer(analyzerName);
-            if (analyzer == null) {
-                throw new IllegalArgumentException("Analyzer [" + analyzerName + "] doesn't exists");
-            }
-            v.setAnalyzer(analyzer);
-        }, Fields.ANALYZER, ObjectParser.ValueType.STRING);
-        TLP_PARSER.declareInt(SuggestionSearchContext.SuggestionContext::setSize, Fields.SIZE);
-        TLP_PARSER.declareInt(SuggestionSearchContext.SuggestionContext::setShardSize, Fields.SHARD_SIZE);
-        TLP_PARSER.declareField((p, v, c) -> {
-            // Copy the current structure. We will parse, once the mapping is provided
-            XContentBuilder builder = XContentFactory.contentBuilder(p.contentType());
-            builder.copyCurrentStructure(p);
-            BytesReference bytes = builder.bytes();
-            c.contextParser = XContentFactory.xContent(bytes).createParser(bytes);
-            p.skipChildren();
-        }, CompletionSuggestionBuilder.CONTEXTS_FIELD, ObjectParser.ValueType.OBJECT); // context is deprecated
-    }
-
-    private static class ContextAndSuggest {
-        XContentParser contextParser;
-        final MapperService mapperService;
-
-        ContextAndSuggest(MapperService mapperService) {
-            this.mapperService = mapperService;
-        }
-    }
-
-    private final CompletionSuggester completionSuggester;
-
-    public CompletionSuggestParser(CompletionSuggester completionSuggester) {
-        this.completionSuggester = completionSuggester;
-    }
-
-    @Override
-    public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException {
-        final CompletionSuggestionContext suggestion = new CompletionSuggestionContext(completionSuggester, mapperService, fieldDataService);
-        final ContextAndSuggest contextAndSuggest = new ContextAndSuggest(mapperService);
-        TLP_PARSER.parse(parser, suggestion, contextAndSuggest);
-        final XContentParser contextParser = contextAndSuggest.contextParser;
-        MappedFieldType mappedFieldType = mapperService.fullName(suggestion.getField());
-        if (mappedFieldType == null) {
-            throw new ElasticsearchException("Field [" + suggestion.getField() + "] is not a completion suggest field");
-        } else if (mappedFieldType instanceof CompletionFieldMapper.CompletionFieldType) {
-            CompletionFieldMapper.CompletionFieldType type = (CompletionFieldMapper.CompletionFieldType) mappedFieldType;
-            if (type.hasContextMappings() == false && contextParser != null) {
-                throw new IllegalArgumentException("suggester [" + type.name() + "] doesn't expect any context");
-            }
-            Map<String, List<ContextMapping.QueryContext>> queryContexts = Collections.emptyMap();
-            if (type.hasContextMappings() && contextParser != null) {
-                ContextMappings contextMappings = type.getContextMappings();
-                contextParser.nextToken();
-                queryContexts = new HashMap<>(contextMappings.size());
-                assert contextParser.currentToken() == XContentParser.Token.START_OBJECT;
-                XContentParser.Token currentToken;
-                String currentFieldName;
-                while ((currentToken = contextParser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                    if (currentToken == XContentParser.Token.FIELD_NAME) {
-                        currentFieldName = contextParser.currentName();
-                        final ContextMapping mapping = contextMappings.get(currentFieldName);
-                        queryContexts.put(currentFieldName, mapping.parseQueryContext(contextParser));
-                    }
-                }
-                contextParser.close();
-            }
-            suggestion.setFieldType(type);
-            suggestion.setQueryContexts(queryContexts);
-            return suggestion;
-        } else {
-            throw new IllegalArgumentException("Field [" + suggestion.getField() + "] is not a completion suggest field");
-        }
-    }
-
-
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
index 527a35658c9..cef0a33fddb 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
@@ -34,10 +34,12 @@ import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.fielddata.AtomicFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.Suggest;
-import org.elasticsearch.search.suggest.SuggestContextParser;
 import org.elasticsearch.search.suggest.Suggester;
+import org.elasticsearch.search.suggest.SuggestionBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -50,9 +52,7 @@ import java.util.Set;
 
 public class CompletionSuggester extends Suggester<CompletionSuggestionContext> {
 
-    public SuggestContextParser getContextParser() {
-        return new CompletionSuggestParser(this);
-    }
+    public static final CompletionSuggester PROTOTYPE = new CompletionSuggester();
 
     @Override
     protected Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>> innerExecute(String name,
@@ -78,15 +78,18 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
             }
             // collect payloads
             final Map<String, List<Object>> payload = new HashMap<>(0);
-            Set<String> payloadFields = suggestionContext.getPayloadFields();
+            List<String> payloadFields = suggestionContext.getPayloadFields();
             if (payloadFields.isEmpty() == false) {
                 final int readerIndex = ReaderUtil.subIndex(suggestDoc.doc, leaves);
                 final LeafReaderContext subReaderContext = leaves.get(readerIndex);
                 final int subDocId = suggestDoc.doc - subReaderContext.docBase;
                 for (String field : payloadFields) {
-                    MappedFieldType payloadFieldType = suggestionContext.getMapperService().fullName(field);
+                    MapperService mapperService = suggestionContext.getShardContext().getMapperService();
+                    MappedFieldType payloadFieldType = mapperService.fullName(field);
                     if (payloadFieldType != null) {
-                        final AtomicFieldData data = suggestionContext.getIndexFieldDataService().getForField(payloadFieldType).load(subReaderContext);
+                        QueryShardContext shardContext = suggestionContext.getShardContext();
+                        final AtomicFieldData data = shardContext.getForField(payloadFieldType)
+                                .load(subReaderContext);
                         final ScriptDocValues scriptValues = data.getScriptValues();
                         scriptValues.setNextDocId(subDocId);
                         payload.put(field, new ArrayList<>(scriptValues.getValues()));
@@ -262,4 +265,9 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
             }
         }
     }
+
+    @Override
+    public SuggestionBuilder<?> getBuilderPrototype() {
+        return CompletionSuggestionBuilder.PROTOTYPE;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
index 9cf78ea6677..ca8aad7c8ac 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
@@ -18,26 +18,37 @@
  */
 package org.elasticsearch.search.suggest.completion;
 
-import org.apache.lucene.search.suggest.document.FuzzyCompletionQuery;
-import org.apache.lucene.util.automaton.Operations;
-import org.apache.lucene.util.automaton.RegExp;
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.query.RegexpFlag;
-import org.elasticsearch.search.suggest.SuggestBuilder;
-import org.elasticsearch.search.suggest.completion.context.CategoryQueryContext;
-import org.elasticsearch.search.suggest.completion.context.GeoQueryContext;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.suggest.SuggestUtils;
+import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
+import org.elasticsearch.search.suggest.completion.context.ContextMapping;
+import org.elasticsearch.search.suggest.completion.context.ContextMappings;
+import org.elasticsearch.search.suggest.completion.context.QueryContext;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
+import java.util.Objects;
 
 /**
  * Defines a suggest command based on a prefix, typically to provide "auto-complete" functionality
@@ -45,218 +56,81 @@ import java.util.Set;
  * are created at index-time and so must be defined in the mapping with the type "completion" before
  * indexing.
  */
-public class CompletionSuggestionBuilder extends SuggestBuilder.SuggestionBuilder<CompletionSuggestionBuilder> {
+public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSuggestionBuilder> {
 
-    final static String SUGGESTION_NAME = "completion";
+    public static final CompletionSuggestionBuilder PROTOTYPE = new CompletionSuggestionBuilder("_na_");
+    static final String SUGGESTION_NAME = "completion";
     static final ParseField PAYLOAD_FIELD = new ParseField("payload");
     static final ParseField CONTEXTS_FIELD = new ParseField("contexts", "context");
-    private FuzzyOptionsBuilder fuzzyOptionsBuilder;
-    private RegexOptionsBuilder regexOptionsBuilder;
-    private final Map<String, List<ToXContent>> queryContexts = new HashMap<>();
-    private final Set<String> payloadFields = new HashSet<>();
 
-    public CompletionSuggestionBuilder(String name) {
-        super(name, SUGGESTION_NAME);
+    /**
+     * {
+     *     "field" : STRING
+     *     "size" : INT
+     *     "fuzzy" : BOOLEAN | FUZZY_OBJECT
+     *     "contexts" : QUERY_CONTEXTS
+     *     "regex" : REGEX_OBJECT
+     *     "payload" : STRING_ARRAY
+     * }
+     */
+    private static ObjectParser<CompletionSuggestionBuilder.InnerBuilder, Void> TLP_PARSER =
+        new ObjectParser<>(SUGGESTION_NAME, null);
+    static {
+        TLP_PARSER.declareStringArray(CompletionSuggestionBuilder.InnerBuilder::payload, PAYLOAD_FIELD);
+        TLP_PARSER.declareField((parser, completionSuggestionContext, context) -> {
+                if (parser.currentToken() == XContentParser.Token.VALUE_BOOLEAN) {
+                    if (parser.booleanValue()) {
+                        completionSuggestionContext.fuzzyOptions = new FuzzyOptions.Builder().build();
+                    }
+                } else {
+                    completionSuggestionContext.fuzzyOptions = FuzzyOptions.parse(parser);
+                }
+            },
+            FuzzyOptions.FUZZY_OPTIONS, ObjectParser.ValueType.OBJECT_OR_BOOLEAN);
+        TLP_PARSER.declareField((parser, completionSuggestionContext, context) ->
+            completionSuggestionContext.regexOptions = RegexOptions.parse(parser),
+            RegexOptions.REGEX_OPTIONS, ObjectParser.ValueType.OBJECT);
+        TLP_PARSER.declareString(CompletionSuggestionBuilder.InnerBuilder::field, SuggestUtils.Fields.FIELD);
+        TLP_PARSER.declareString(CompletionSuggestionBuilder.InnerBuilder::analyzer, SuggestUtils.Fields.ANALYZER);
+        TLP_PARSER.declareInt(CompletionSuggestionBuilder.InnerBuilder::size, SuggestUtils.Fields.SIZE);
+        TLP_PARSER.declareInt(CompletionSuggestionBuilder.InnerBuilder::shardSize, SuggestUtils.Fields.SHARD_SIZE);
+        TLP_PARSER.declareField((p, v, c) -> {
+            // Copy the current structure. We will parse, once the mapping is provided
+            XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+            builder.copyCurrentStructure(p);
+            v.contextBytes = builder.bytes();
+            p.skipChildren();
+        }, CONTEXTS_FIELD, ObjectParser.ValueType.OBJECT); // context is deprecated
+    }
+
+    protected FuzzyOptions fuzzyOptions;
+    protected RegexOptions regexOptions;
+    protected BytesReference contextBytes = null;
+    protected List<String> payloadFields = Collections.emptyList();
+
+    public CompletionSuggestionBuilder(String field) {
+        super(field);
     }
 
     /**
-     * Options for fuzzy queries
+     * internal copy constructor that copies over all class fields except for the field which is
+     * set to the one provided in the first argument
      */
-    public static class FuzzyOptionsBuilder implements ToXContent {
-        static final ParseField FUZZY_OPTIONS = new ParseField("fuzzy");
-        static final ParseField TRANSPOSITION_FIELD = new ParseField("transpositions");
-        static final ParseField MIN_LENGTH_FIELD = new ParseField("min_length");
-        static final ParseField PREFIX_LENGTH_FIELD = new ParseField("prefix_length");
-        static final ParseField UNICODE_AWARE_FIELD = new ParseField("unicode_aware");
-        static final ParseField MAX_DETERMINIZED_STATES_FIELD = new ParseField("max_determinized_states");
-
-        private int editDistance = FuzzyCompletionQuery.DEFAULT_MAX_EDITS;
-        private boolean transpositions = FuzzyCompletionQuery.DEFAULT_TRANSPOSITIONS;
-        private int fuzzyMinLength = FuzzyCompletionQuery.DEFAULT_MIN_FUZZY_LENGTH;
-        private int fuzzyPrefixLength = FuzzyCompletionQuery.DEFAULT_NON_FUZZY_PREFIX;
-        private boolean unicodeAware = FuzzyCompletionQuery.DEFAULT_UNICODE_AWARE;
-        private int maxDeterminizedStates = Operations.DEFAULT_MAX_DETERMINIZED_STATES;
-
-        public FuzzyOptionsBuilder() {
-        }
-
-        /**
-         * Sets the level of fuzziness used to create suggestions using a {@link Fuzziness} instance.
-         * The default value is {@link Fuzziness#ONE} which allows for an "edit distance" of one.
-         */
-        public FuzzyOptionsBuilder setFuzziness(int editDistance) {
-            this.editDistance = editDistance;
-            return this;
-        }
-
-        /**
-         * Sets the level of fuzziness used to create suggestions using a {@link Fuzziness} instance.
-         * The default value is {@link Fuzziness#ONE} which allows for an "edit distance" of one.
-         */
-        public FuzzyOptionsBuilder setFuzziness(Fuzziness fuzziness) {
-            this.editDistance = fuzziness.asDistance();
-            return this;
-        }
-
-        /**
-         * Sets if transpositions (swapping one character for another) counts as one character
-         * change or two.
-         * Defaults to true, meaning it uses the fuzzier option of counting transpositions as
-         * a single change.
-         */
-        public FuzzyOptionsBuilder setTranspositions(boolean transpositions) {
-            this.transpositions = transpositions;
-            return this;
-        }
-
-        /**
-         * Sets the minimum length of input string before fuzzy suggestions are returned, defaulting
-         * to 3.
-         */
-        public FuzzyOptionsBuilder setFuzzyMinLength(int fuzzyMinLength) {
-            this.fuzzyMinLength = fuzzyMinLength;
-            return this;
-        }
-
-        /**
-         * Sets the minimum length of the input, which is not checked for fuzzy alternatives, defaults to 1
-         */
-        public FuzzyOptionsBuilder setFuzzyPrefixLength(int fuzzyPrefixLength) {
-            this.fuzzyPrefixLength = fuzzyPrefixLength;
-            return this;
-        }
-
-        /**
-         * Sets the maximum automaton states allowed for the fuzzy expansion
-         */
-        public FuzzyOptionsBuilder setMaxDeterminizedStates(int maxDeterminizedStates) {
-            this.maxDeterminizedStates = maxDeterminizedStates;
-            return this;
-        }
-
-        /**
-         * Set to true if all measurements (like edit distance, transpositions and lengths) are in unicode
-         * code points (actual letters) instead of bytes. Default is false.
-         */
-        public FuzzyOptionsBuilder setUnicodeAware(boolean unicodeAware) {
-            this.unicodeAware = unicodeAware;
-            return this;
-        }
-
-        /**
-         * Returns the maximum number of edits
-         */
-        int getEditDistance() {
-            return editDistance;
-        }
-
-        /**
-         * Returns if transpositions option is set
-         *
-         * if transpositions is set, then swapping one character for another counts as one edit instead of two.
-         */
-        boolean isTranspositions() {
-            return transpositions;
-        }
-
-
-        /**
-         * Returns the length of input prefix after which edits are applied
-         */
-        int getFuzzyMinLength() {
-            return fuzzyMinLength;
-        }
-
-        /**
-         * Returns the minimum length of the input prefix required to apply any edits
-         */
-        int getFuzzyPrefixLength() {
-            return fuzzyPrefixLength;
-        }
-
-        /**
-         * Returns if all measurements (like edit distance, transpositions and lengths) are in unicode code
-         * points (actual letters) instead of bytes.
-         */
-        boolean isUnicodeAware() {
-            return unicodeAware;
-        }
-
-        /**
-         * Returns the maximum automaton states allowed for fuzzy expansion
-         */
-        int getMaxDeterminizedStates() {
-            return maxDeterminizedStates;
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(FUZZY_OPTIONS.getPreferredName());
-            builder.field(Fuzziness.FIELD.getPreferredName(), editDistance);
-            builder.field(TRANSPOSITION_FIELD.getPreferredName(), transpositions);
-            builder.field(MIN_LENGTH_FIELD.getPreferredName(), fuzzyMinLength);
-            builder.field(PREFIX_LENGTH_FIELD.getPreferredName(), fuzzyPrefixLength);
-            builder.field(UNICODE_AWARE_FIELD.getPreferredName(), unicodeAware);
-            builder.field(MAX_DETERMINIZED_STATES_FIELD.getPreferredName(), maxDeterminizedStates);
-            builder.endObject();
-            return builder;
-        }
-    }
-
-    /**
-     * Options for regular expression queries
-     */
-    public static class RegexOptionsBuilder implements ToXContent {
-        static final ParseField REGEX_OPTIONS = new ParseField("regex");
-        static final ParseField FLAGS_VALUE = new ParseField("flags", "flags_value");
-        static final ParseField MAX_DETERMINIZED_STATES = new ParseField("max_determinized_states");
-        private int flagsValue = RegExp.ALL;
-        private int maxDeterminizedStates = Operations.DEFAULT_MAX_DETERMINIZED_STATES;
-
-        public RegexOptionsBuilder() {
-        }
-
-        /**
-         * Sets the regular expression syntax flags
-         * see {@link RegexpFlag}
-         */
-        public RegexOptionsBuilder setFlags(String flags) {
-            this.flagsValue = RegexpFlag.resolveValue(flags);
-            return this;
-        }
-
-        /**
-         * Sets the maximum automaton states allowed for the regular expression expansion
-         */
-        public RegexOptionsBuilder setMaxDeterminizedStates(int maxDeterminizedStates) {
-            this.maxDeterminizedStates = maxDeterminizedStates;
-            return this;
-        }
-
-        int getFlagsValue() {
-            return flagsValue;
-        }
-
-        int getMaxDeterminizedStates() {
-            return maxDeterminizedStates;
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(REGEX_OPTIONS.getPreferredName());
-            builder.field(FLAGS_VALUE.getPreferredName(), flagsValue);
-            builder.field(MAX_DETERMINIZED_STATES.getPreferredName(), maxDeterminizedStates);
-            builder.endObject();
-            return builder;
-        }
+    private CompletionSuggestionBuilder(String fieldname, CompletionSuggestionBuilder in) {
+        super(fieldname, in);
+        fuzzyOptions = in.fuzzyOptions;
+        regexOptions = in.regexOptions;
+        contextBytes = in.contextBytes;
+        payloadFields = in.payloadFields;
     }
 
     /**
      * Sets the prefix to provide completions for.
      * The prefix gets analyzed by the suggest analyzer.
      */
+    @Override
     public CompletionSuggestionBuilder prefix(String prefix) {
-        super.setPrefix(prefix);
+        super.prefix(prefix);
         return this;
     }
 
@@ -264,36 +138,37 @@ public class CompletionSuggestionBuilder extends SuggestBuilder.SuggestionBuilde
      * Same as {@link #prefix(String)} with fuzziness of <code>fuzziness</code>
      */
     public CompletionSuggestionBuilder prefix(String prefix, Fuzziness fuzziness) {
-        super.setPrefix(prefix);
-        this.fuzzyOptionsBuilder = new FuzzyOptionsBuilder().setFuzziness(fuzziness);
+        super.prefix(prefix);
+        this.fuzzyOptions = new FuzzyOptions.Builder().setFuzziness(fuzziness).build();
         return this;
     }
 
     /**
      * Same as {@link #prefix(String)} with full fuzzy options
-     * see {@link FuzzyOptionsBuilder}
+     * see {@link FuzzyOptions.Builder}
      */
-    public CompletionSuggestionBuilder prefix(String prefix, FuzzyOptionsBuilder fuzzyOptionsBuilder) {
-        super.setPrefix(prefix);
-        this.fuzzyOptionsBuilder = fuzzyOptionsBuilder;
+    public CompletionSuggestionBuilder prefix(String prefix, FuzzyOptions fuzzyOptions) {
+        super.prefix(prefix);
+        this.fuzzyOptions = fuzzyOptions;
         return this;
     }
 
     /**
      * Sets a regular expression pattern for prefixes to provide completions for.
      */
+    @Override
     public CompletionSuggestionBuilder regex(String regex) {
-        super.setRegex(regex);
+        super.regex(regex);
         return this;
     }
 
     /**
      * Same as {@link #regex(String)} with full regular expression options
-     * see {@link RegexOptionsBuilder}
+     * see {@link RegexOptions.Builder}
      */
-    public CompletionSuggestionBuilder regex(String regex, RegexOptionsBuilder regexOptionsBuilder) {
+    public CompletionSuggestionBuilder regex(String regex, RegexOptions regexOptions) {
         this.regex(regex);
-        this.regexOptionsBuilder = regexOptionsBuilder;
+        this.regexOptions = regexOptions;
         return this;
     }
 
@@ -301,65 +176,189 @@ public class CompletionSuggestionBuilder extends SuggestBuilder.SuggestionBuilde
      * Sets the fields to be returned as suggestion payload.
      * Note: Only doc values enabled fields are supported
      */
-    public CompletionSuggestionBuilder payload(String... fields) {
-        Collections.addAll(this.payloadFields, fields);
+    public CompletionSuggestionBuilder payload(List<String> fields) {
+        Objects.requireNonNull(fields, "payload must not be null");
+        this.payloadFields = fields;
         return this;
     }
 
     /**
-     * Sets query contexts for a category context
-     * @param name of the category context to execute on
-     * @param queryContexts a list of {@link CategoryQueryContext}
+     * Sets query contexts for completion
+     * @param queryContexts named query contexts
+     *                      see {@link org.elasticsearch.search.suggest.completion.context.CategoryQueryContext}
+     *                      and {@link org.elasticsearch.search.suggest.completion.context.GeoQueryContext}
      */
-    public CompletionSuggestionBuilder categoryContexts(String name, CategoryQueryContext... queryContexts) {
-        return contexts(name, queryContexts);
-    }
-
-    /**
-     * Sets query contexts for a geo context
-     * @param name of the geo context to execute on
-     * @param queryContexts a list of {@link GeoQueryContext}
-     */
-    public CompletionSuggestionBuilder geoContexts(String name, GeoQueryContext... queryContexts) {
-        return contexts(name, queryContexts);
-    }
-
-    private CompletionSuggestionBuilder contexts(String name, ToXContent... queryContexts) {
-        List<ToXContent> contexts = this.queryContexts.get(name);
-        if (contexts == null) {
-            contexts = new ArrayList<>(2);
-            this.queryContexts.put(name, contexts);
+    public CompletionSuggestionBuilder contexts(Map<String, List<? extends QueryContext>> queryContexts) {
+        Objects.requireNonNull(queryContexts, "contexts must not be null");
+        try {
+            XContentBuilder contentBuilder = XContentFactory.jsonBuilder();
+            contentBuilder.startObject();
+            for (Map.Entry<String, List<? extends QueryContext>> contextEntry : queryContexts.entrySet()) {
+                contentBuilder.startArray(contextEntry.getKey());
+                for (ToXContent queryContext : contextEntry.getValue()) {
+                    queryContext.toXContent(contentBuilder, EMPTY_PARAMS);
+                }
+                contentBuilder.endArray();
+            }
+            contentBuilder.endObject();
+            contextBytes = contentBuilder.bytes();
+            return this;
+        } catch (IOException e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+
+    private static class InnerBuilder extends CompletionSuggestionBuilder {
+        private String field;
+
+        public InnerBuilder() {
+            super("_na_");
+        }
+
+        private InnerBuilder field(String field) {
+            this.field = field;
+            return this;
         }
-        Collections.addAll(contexts, queryContexts);
-        return this;
     }
 
     @Override
     protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
-        if (payloadFields != null) {
+        if (payloadFields.isEmpty() == false) {
             builder.startArray(PAYLOAD_FIELD.getPreferredName());
             for (String field : payloadFields) {
                 builder.value(field);
             }
             builder.endArray();
         }
-        if (fuzzyOptionsBuilder != null) {
-            fuzzyOptionsBuilder.toXContent(builder, params);
+        if (fuzzyOptions != null) {
+            fuzzyOptions.toXContent(builder, params);
         }
-        if (regexOptionsBuilder != null) {
-            regexOptionsBuilder.toXContent(builder, params);
+        if (regexOptions != null) {
+            regexOptions.toXContent(builder, params);
         }
-        if (queryContexts.isEmpty() == false) {
-            builder.startObject(CONTEXTS_FIELD.getPreferredName());
-            for (Map.Entry<String, List<ToXContent>> entry : this.queryContexts.entrySet()) {
-                builder.startArray(entry.getKey());
-                for (ToXContent queryContext : entry.getValue()) {
-                    queryContext.toXContent(builder, params);
-                }
-                builder.endArray();
-            }
-            builder.endObject();
+        if (contextBytes != null) {
+            XContentParser contextParser = XContentFactory.xContent(XContentType.JSON).createParser(contextBytes);
+            builder.field(CONTEXTS_FIELD.getPreferredName());
+            builder.copyCurrentStructure(contextParser);
         }
         return builder;
     }
+
+    @Override
+    protected CompletionSuggestionBuilder innerFromXContent(QueryParseContext parseContext) throws IOException {
+        CompletionSuggestionBuilder.InnerBuilder builder = new CompletionSuggestionBuilder.InnerBuilder();
+        TLP_PARSER.parse(parseContext.parser(), builder);
+        String field = builder.field;
+        // now we should have field name, check and copy fields over to the suggestion builder we return
+        if (field == null) {
+            throw new ElasticsearchParseException(
+                "the required field option [" + SuggestUtils.Fields.FIELD.getPreferredName() + "] is missing");
+        }
+        return new CompletionSuggestionBuilder(field, builder);
+    }
+
+    @Override
+    public SuggestionContext build(QueryShardContext context) throws IOException {
+        CompletionSuggestionContext suggestionContext = new CompletionSuggestionContext(context);
+        // copy over common settings to each suggestion builder
+        final MapperService mapperService = context.getMapperService();
+        populateCommonFields(mapperService, suggestionContext);
+        suggestionContext.setPayloadFields(payloadFields);
+        suggestionContext.setFuzzyOptions(fuzzyOptions);
+        suggestionContext.setRegexOptions(regexOptions);
+        MappedFieldType mappedFieldType = mapperService.fullName(suggestionContext.getField());
+        if (mappedFieldType != null && mappedFieldType instanceof CompletionFieldMapper.CompletionFieldType) {
+            CompletionFieldMapper.CompletionFieldType type = (CompletionFieldMapper.CompletionFieldType) mappedFieldType;
+            suggestionContext.setFieldType(type);
+            if (type.hasContextMappings() && contextBytes != null) {
+                try (XContentParser contextParser = XContentFactory.xContent(contextBytes).createParser(contextBytes)) {
+                    if (type.hasContextMappings() && contextParser != null) {
+                        ContextMappings contextMappings = type.getContextMappings();
+                        contextParser.nextToken();
+                        Map<String, List<ContextMapping.InternalQueryContext>> queryContexts = new HashMap<>(contextMappings.size());
+                        assert contextParser.currentToken() == XContentParser.Token.START_OBJECT;
+                        XContentParser.Token currentToken;
+                        String currentFieldName;
+                        while ((currentToken = contextParser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                            if (currentToken == XContentParser.Token.FIELD_NAME) {
+                                currentFieldName = contextParser.currentName();
+                                final ContextMapping mapping = contextMappings.get(currentFieldName);
+                                queryContexts.put(currentFieldName, mapping.parseQueryContext(contextParser));
+                            }
+                        }
+                        suggestionContext.setQueryContexts(queryContexts);
+                    }
+                }
+            } else if (contextBytes != null) {
+                throw new IllegalArgumentException("suggester [" + type.name() + "] doesn't expect any context");
+            }
+        } else {
+            throw new IllegalArgumentException("Field [" + suggestionContext.getField() + "] is not a completion suggest field");
+        }
+        return suggestionContext;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return SUGGESTION_NAME;
+    }
+
+    @Override
+    public void doWriteTo(StreamOutput out) throws IOException {
+        out.writeBoolean(payloadFields.isEmpty() == false);
+        if (payloadFields.isEmpty() == false) {
+            out.writeVInt(payloadFields.size());
+            for (String payloadField : payloadFields) {
+                out.writeString(payloadField);
+            }
+        }
+        out.writeBoolean(fuzzyOptions != null);
+        if (fuzzyOptions != null) {
+            fuzzyOptions.writeTo(out);
+        }
+        out.writeBoolean(regexOptions != null);
+        if (regexOptions != null) {
+            regexOptions.writeTo(out);
+        }
+        out.writeBoolean(contextBytes != null);
+        if (contextBytes != null) {
+            out.writeBytesReference(contextBytes);
+        }
+    }
+
+    @Override
+    public CompletionSuggestionBuilder doReadFrom(StreamInput in, String field) throws IOException {
+        CompletionSuggestionBuilder completionSuggestionBuilder = new CompletionSuggestionBuilder(field);
+        if (in.readBoolean()) {
+            int numPayloadField = in.readVInt();
+            List<String> payloadFields = new ArrayList<>(numPayloadField);
+            for (int i = 0; i < numPayloadField; i++) {
+                payloadFields.add(in.readString());
+            }
+            completionSuggestionBuilder.payloadFields = payloadFields;
+        }
+        if (in.readBoolean()) {
+            completionSuggestionBuilder.fuzzyOptions = FuzzyOptions.readFuzzyOptions(in);
+        }
+        if (in.readBoolean()) {
+            completionSuggestionBuilder.regexOptions = RegexOptions.readRegexOptions(in);
+        }
+        if (in.readBoolean()) {
+            completionSuggestionBuilder.contextBytes = in.readBytesReference();
+        }
+        return completionSuggestionBuilder;
+    }
+
+    @Override
+    protected boolean doEquals(CompletionSuggestionBuilder other) {
+        return Objects.equals(payloadFields, other.payloadFields) &&
+            Objects.equals(fuzzyOptions, other.fuzzyOptions) &&
+            Objects.equals(regexOptions, other.regexOptions) &&
+            Objects.equals(contextBytes, other.contextBytes);
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(payloadFields, fuzzyOptions, regexOptions, contextBytes);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java
index 535151b476e..f9f948a551f 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java
@@ -20,39 +20,31 @@ package org.elasticsearch.search.suggest.completion;
 
 import org.apache.lucene.search.suggest.document.CompletionQuery;
 import org.elasticsearch.common.unit.Fuzziness;
-import org.elasticsearch.index.fielddata.IndexFieldDataService;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
-import org.elasticsearch.search.suggest.Suggester;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
 import org.elasticsearch.search.suggest.completion.context.ContextMapping;
 import org.elasticsearch.search.suggest.completion.context.ContextMappings;
 
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 /**
  *
  */
 public class CompletionSuggestionContext extends SuggestionSearchContext.SuggestionContext {
 
-    private CompletionFieldMapper.CompletionFieldType fieldType;
-    private CompletionSuggestionBuilder.FuzzyOptionsBuilder fuzzyOptionsBuilder;
-    private CompletionSuggestionBuilder.RegexOptionsBuilder regexOptionsBuilder;
-    private Map<String, List<ContextMapping.QueryContext>> queryContexts = Collections.emptyMap();
-    private final MapperService mapperService;
-    private final IndexFieldDataService indexFieldDataService;
-    private Set<String> payloadFields = Collections.emptySet();
-
-    CompletionSuggestionContext(Suggester suggester, MapperService mapperService, IndexFieldDataService indexFieldDataService) {
-        super(suggester);
-        this.indexFieldDataService = indexFieldDataService;
-        this.mapperService = mapperService;
+    protected CompletionSuggestionContext(QueryShardContext shardContext) {
+        super(CompletionSuggester.PROTOTYPE, shardContext);
     }
 
+    private CompletionFieldMapper.CompletionFieldType fieldType;
+    private FuzzyOptions fuzzyOptions;
+    private RegexOptions regexOptions;
+    private Map<String, List<ContextMapping.InternalQueryContext>> queryContexts = Collections.emptyMap();
+    private List<String> payloadFields = Collections.emptyList();
+
     CompletionFieldMapper.CompletionFieldType getFieldType() {
         return this.fieldType;
     }
@@ -61,61 +53,60 @@ public class CompletionSuggestionContext extends SuggestionSearchContext.Suggest
         this.fieldType = fieldType;
     }
 
-    void setRegexOptionsBuilder(CompletionSuggestionBuilder.RegexOptionsBuilder regexOptionsBuilder) {
-        this.regexOptionsBuilder = regexOptionsBuilder;
+    void setRegexOptions(RegexOptions regexOptions) {
+        this.regexOptions = regexOptions;
     }
 
-    void setFuzzyOptionsBuilder(CompletionSuggestionBuilder.FuzzyOptionsBuilder fuzzyOptionsBuilder) {
-        this.fuzzyOptionsBuilder = fuzzyOptionsBuilder;
+    void setFuzzyOptions(FuzzyOptions fuzzyOptions) {
+        this.fuzzyOptions = fuzzyOptions;
     }
 
-    void setQueryContexts(Map<String, List<ContextMapping.QueryContext>> queryContexts) {
+    void setQueryContexts(Map<String, List<ContextMapping.InternalQueryContext>> queryContexts) {
         this.queryContexts = queryContexts;
     }
 
-
-    MapperService getMapperService() {
-        return mapperService;
-    }
-
-    IndexFieldDataService getIndexFieldDataService() {
-        return indexFieldDataService;
-    }
-
-    void setPayloadFields(Set<String> fields) {
+    void setPayloadFields(List<String> fields) {
         this.payloadFields = fields;
     }
 
-    void setPayloadFields(List<String> fields) {
-        setPayloadFields(new HashSet<String>(fields));
+    List<String> getPayloadFields() {
+        return payloadFields;
     }
 
-    Set<String> getPayloadFields() {
-        return payloadFields;
+    public FuzzyOptions getFuzzyOptions() {
+        return fuzzyOptions;
+    }
+
+    public RegexOptions getRegexOptions() {
+        return regexOptions;
+    }
+
+    public Map<String, List<ContextMapping.InternalQueryContext>> getQueryContexts() {
+        return queryContexts;
     }
 
     CompletionQuery toQuery() {
         CompletionFieldMapper.CompletionFieldType fieldType = getFieldType();
         final CompletionQuery query;
         if (getPrefix() != null) {
-            if (fuzzyOptionsBuilder != null) {
+            if (fuzzyOptions != null) {
                 query = fieldType.fuzzyQuery(getPrefix().utf8ToString(),
-                        Fuzziness.fromEdits(fuzzyOptionsBuilder.getEditDistance()),
-                        fuzzyOptionsBuilder.getFuzzyPrefixLength(), fuzzyOptionsBuilder.getFuzzyMinLength(),
-                        fuzzyOptionsBuilder.getMaxDeterminizedStates(), fuzzyOptionsBuilder.isTranspositions(),
-                        fuzzyOptionsBuilder.isUnicodeAware());
+                        Fuzziness.fromEdits(fuzzyOptions.getEditDistance()),
+                        fuzzyOptions.getFuzzyPrefixLength(), fuzzyOptions.getFuzzyMinLength(),
+                        fuzzyOptions.getMaxDeterminizedStates(), fuzzyOptions.isTranspositions(),
+                        fuzzyOptions.isUnicodeAware());
             } else {
                 query = fieldType.prefixQuery(getPrefix());
             }
         } else if (getRegex() != null) {
-            if (fuzzyOptionsBuilder != null) {
+            if (fuzzyOptions != null) {
                 throw new IllegalArgumentException("can not use 'fuzzy' options with 'regex");
             }
-            if (regexOptionsBuilder == null) {
-                regexOptionsBuilder = new CompletionSuggestionBuilder.RegexOptionsBuilder();
+            if (regexOptions == null) {
+                regexOptions = RegexOptions.builder().build();
             }
-            query = fieldType.regexpQuery(getRegex(), regexOptionsBuilder.getFlagsValue(),
-                    regexOptionsBuilder.getMaxDeterminizedStates());
+            query = fieldType.regexpQuery(getRegex(), regexOptions.getFlagsValue(),
+                    regexOptions.getMaxDeterminizedStates());
         } else {
             throw new IllegalArgumentException("'prefix' or 'regex' must be defined");
         }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/FuzzyOptions.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/FuzzyOptions.java
new file mode 100644
index 00000000000..8f05be04699
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/FuzzyOptions.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.completion;
+
+import org.apache.lucene.search.suggest.document.FuzzyCompletionQuery;
+import org.apache.lucene.util.automaton.Operations;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Fuzzy options for completion suggester
+ */
+public class FuzzyOptions implements ToXContent, Writeable<FuzzyOptions> {
+    static final ParseField FUZZY_OPTIONS = new ParseField("fuzzy");
+    private static final ParseField TRANSPOSITION_FIELD = new ParseField("transpositions");
+    private static final ParseField MIN_LENGTH_FIELD = new ParseField("min_length");
+    private static final ParseField PREFIX_LENGTH_FIELD = new ParseField("prefix_length");
+    private static final ParseField UNICODE_AWARE_FIELD = new ParseField("unicode_aware");
+    private static final ParseField MAX_DETERMINIZED_STATES_FIELD = new ParseField("max_determinized_states");
+
+    /**
+     * fuzzy : {
+     *     "edit_distance" : STRING | INT
+     *     "transpositions" : BOOLEAN
+     *     "min_length" : INT
+     *     "prefix_length" : INT
+     *     "unicode_aware" : BOOLEAN
+     *     "max_determinized_states" : INT
+     * }
+     */
+    private static ObjectParser<Builder, Void> PARSER = new ObjectParser<>(FUZZY_OPTIONS.getPreferredName(), Builder::new);
+    static {
+        PARSER.declareInt(Builder::setFuzzyMinLength, MIN_LENGTH_FIELD);
+        PARSER.declareInt(Builder::setMaxDeterminizedStates, MAX_DETERMINIZED_STATES_FIELD);
+        PARSER.declareBoolean(Builder::setUnicodeAware, UNICODE_AWARE_FIELD);
+        PARSER.declareInt(Builder::setFuzzyPrefixLength, PREFIX_LENGTH_FIELD);
+        PARSER.declareBoolean(Builder::setTranspositions, TRANSPOSITION_FIELD);
+        PARSER.declareValue((a, b) -> {
+            try {
+                a.setFuzziness(Fuzziness.parse(b).asDistance());
+            } catch (IOException e) {
+                throw new ElasticsearchException(e);
+            }
+        }, Fuzziness.FIELD);
+    }
+
+    private int editDistance;
+    private boolean transpositions;
+    private int fuzzyMinLength;
+    private int fuzzyPrefixLength;
+    private boolean unicodeAware;
+    private int maxDeterminizedStates;
+
+    private FuzzyOptions(int editDistance, boolean transpositions, int fuzzyMinLength, int fuzzyPrefixLength,
+                         boolean unicodeAware, int maxDeterminizedStates) {
+        this.editDistance = editDistance;
+        this.transpositions = transpositions;
+        this.fuzzyMinLength = fuzzyMinLength;
+        this.fuzzyPrefixLength = fuzzyPrefixLength;
+        this.unicodeAware = unicodeAware;
+        this.maxDeterminizedStates = maxDeterminizedStates;
+    }
+
+    private FuzzyOptions() {
+    }
+
+    static FuzzyOptions parse(XContentParser parser) throws IOException {
+        return PARSER.parse(parser).build();
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    /**
+     * Returns the maximum number of edits
+     */
+    public int getEditDistance() {
+        return editDistance;
+    }
+
+    /**
+     * Returns if transpositions option is set
+     *
+     * if transpositions is set, then swapping one character for another counts as one edit instead of two.
+     */
+    public boolean isTranspositions() {
+        return transpositions;
+    }
+
+    /**
+     * Returns the length of input prefix after which edits are applied
+     */
+    public int getFuzzyMinLength() {
+        return fuzzyMinLength;
+    }
+
+    /**
+     * Returns the minimum length of the input prefix required to apply any edits
+     */
+    public int getFuzzyPrefixLength() {
+        return fuzzyPrefixLength;
+    }
+
+    /**
+     * Returns if all measurements (like edit distance, transpositions and lengths) are in unicode code
+     * points (actual letters) instead of bytes.
+     */
+    public boolean isUnicodeAware() {
+        return unicodeAware;
+    }
+
+    /**
+     * Returns the maximum automaton states allowed for fuzzy expansion
+     */
+    public int getMaxDeterminizedStates() {
+        return maxDeterminizedStates;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        FuzzyOptions that = (FuzzyOptions) o;
+
+        if (editDistance != that.editDistance) return false;
+        if (transpositions != that.transpositions) return false;
+        if (fuzzyMinLength != that.fuzzyMinLength) return false;
+        if (fuzzyPrefixLength != that.fuzzyPrefixLength) return false;
+        if (unicodeAware != that.unicodeAware) return false;
+        return maxDeterminizedStates == that.maxDeterminizedStates;
+
+    }
+
+    @Override
+    public int hashCode() {
+        int result = editDistance;
+        result = 31 * result + (transpositions ? 1 : 0);
+        result = 31 * result + fuzzyMinLength;
+        result = 31 * result + fuzzyPrefixLength;
+        result = 31 * result + (unicodeAware ? 1 : 0);
+        result = 31 * result + maxDeterminizedStates;
+        return result;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(FUZZY_OPTIONS.getPreferredName());
+        builder.field(Fuzziness.FIELD.getPreferredName(), editDistance);
+        builder.field(TRANSPOSITION_FIELD.getPreferredName(), transpositions);
+        builder.field(MIN_LENGTH_FIELD.getPreferredName(), fuzzyMinLength);
+        builder.field(PREFIX_LENGTH_FIELD.getPreferredName(), fuzzyPrefixLength);
+        builder.field(UNICODE_AWARE_FIELD.getPreferredName(), unicodeAware);
+        builder.field(MAX_DETERMINIZED_STATES_FIELD.getPreferredName(), maxDeterminizedStates);
+        builder.endObject();
+        return builder;
+    }
+
+    public static FuzzyOptions readFuzzyOptions(StreamInput in) throws IOException {
+        FuzzyOptions fuzzyOptions = new FuzzyOptions();
+        fuzzyOptions.readFrom(in);
+        return fuzzyOptions;
+    }
+
+    @Override
+    public FuzzyOptions readFrom(StreamInput in) throws IOException {
+        this.transpositions = in.readBoolean();
+        this.unicodeAware = in.readBoolean();
+        this.editDistance = in.readVInt();
+        this.fuzzyMinLength = in.readVInt();
+        this.fuzzyPrefixLength = in.readVInt();
+        this.maxDeterminizedStates = in.readVInt();
+        return this;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeBoolean(transpositions);
+        out.writeBoolean(unicodeAware);
+        out.writeVInt(editDistance);
+        out.writeVInt(fuzzyMinLength);
+        out.writeVInt(fuzzyPrefixLength);
+        out.writeVInt(maxDeterminizedStates);
+    }
+
+    /**
+     * Options for fuzzy queries
+     */
+    public static class Builder {
+
+        private int editDistance = FuzzyCompletionQuery.DEFAULT_MAX_EDITS;
+        private boolean transpositions = FuzzyCompletionQuery.DEFAULT_TRANSPOSITIONS;
+        private int fuzzyMinLength = FuzzyCompletionQuery.DEFAULT_MIN_FUZZY_LENGTH;
+        private int fuzzyPrefixLength = FuzzyCompletionQuery.DEFAULT_NON_FUZZY_PREFIX;
+        private boolean unicodeAware = FuzzyCompletionQuery.DEFAULT_UNICODE_AWARE;
+        private int maxDeterminizedStates = Operations.DEFAULT_MAX_DETERMINIZED_STATES;
+
+        public Builder() {
+        }
+
+        /**
+         * Sets the level of fuzziness used to create suggestions using a {@link Fuzziness} instance.
+         * The default value is {@link Fuzziness#ONE} which allows for an "edit distance" of one.
+         */
+        public Builder setFuzziness(int editDistance) {
+            if (editDistance < 0 || editDistance > 2) {
+                throw new IllegalArgumentException("fuzziness must be between 0 and 2");
+            }
+            this.editDistance = editDistance;
+            return this;
+        }
+
+        /**
+         * Sets the level of fuzziness used to create suggestions using a {@link Fuzziness} instance.
+         * The default value is {@link Fuzziness#ONE} which allows for an "edit distance" of one.
+         */
+        public Builder setFuzziness(Fuzziness fuzziness) {
+            Objects.requireNonNull(fuzziness, "fuzziness must not be null");
+            return setFuzziness(fuzziness.asDistance());
+        }
+
+        /**
+         * Sets if transpositions (swapping one character for another) counts as one character
+         * change or two.
+         * Defaults to true, meaning it uses the fuzzier option of counting transpositions as
+         * a single change.
+         */
+        public Builder setTranspositions(boolean transpositions) {
+            this.transpositions = transpositions;
+            return this;
+        }
+
+        /**
+         * Sets the minimum length of input string before fuzzy suggestions are returned, defaulting
+         * to 3.
+         */
+        public Builder setFuzzyMinLength(int fuzzyMinLength) {
+            if (fuzzyMinLength < 0) {
+                throw new IllegalArgumentException("fuzzyMinLength must not be negative");
+            }
+            this.fuzzyMinLength = fuzzyMinLength;
+            return this;
+        }
+
+        /**
+         * Sets the minimum length of the input, which is not checked for fuzzy alternatives, defaults to 1
+         */
+        public Builder setFuzzyPrefixLength(int fuzzyPrefixLength) {
+            if (fuzzyPrefixLength < 0) {
+                throw new IllegalArgumentException("fuzzyPrefixLength must not be negative");
+            }
+            this.fuzzyPrefixLength = fuzzyPrefixLength;
+            return this;
+        }
+
+        /**
+         * Sets the maximum automaton states allowed for the fuzzy expansion
+         */
+        public Builder setMaxDeterminizedStates(int maxDeterminizedStates) {
+            if (maxDeterminizedStates < 0) {
+                throw new IllegalArgumentException("maxDeterminizedStates must not be negative");
+            }
+            this.maxDeterminizedStates = maxDeterminizedStates;
+            return this;
+        }
+
+        /**
+         * Set to true if all measurements (like edit distance, transpositions and lengths) are in unicode
+         * code points (actual letters) instead of bytes. Default is false.
+         */
+        public Builder setUnicodeAware(boolean unicodeAware) {
+            this.unicodeAware = unicodeAware;
+            return this;
+        }
+
+        public FuzzyOptions build() {
+            return new FuzzyOptions(editDistance, transpositions, fuzzyMinLength, fuzzyPrefixLength,
+                unicodeAware, maxDeterminizedStates);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/RegexOptions.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/RegexOptions.java
new file mode 100644
index 00000000000..8503dbdf46d
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/RegexOptions.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.completion;
+
+import org.apache.lucene.util.automaton.Operations;
+import org.apache.lucene.util.automaton.RegExp;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.RegexpFlag;
+
+import java.io.IOException;
+
+/**
+ * Regular expression options for completion suggester
+ */
+public class RegexOptions implements ToXContent, Writeable<RegexOptions> {
+    static final ParseField REGEX_OPTIONS = new ParseField("regex");
+    private static final ParseField FLAGS_VALUE = new ParseField("flags", "flags_value");
+    private static final ParseField MAX_DETERMINIZED_STATES = new ParseField("max_determinized_states");
+
+    /**
+     * regex: {
+     *     "flags" : STRING | INT
+     *     "max_determinized_states" : INT
+     * }
+     */
+    private static ObjectParser<Builder, Void> PARSER = new ObjectParser<>(REGEX_OPTIONS.getPreferredName(), Builder::new);
+    static {
+        PARSER.declareInt(Builder::setMaxDeterminizedStates, MAX_DETERMINIZED_STATES);
+        PARSER.declareField((parser, builder, aVoid) -> {
+            if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
+                builder.setFlags(parser.text());
+            } else if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
+                builder.setFlagsValue(parser.intValue());
+            } else {
+                throw new ElasticsearchParseException(REGEX_OPTIONS.getPreferredName()
+                    + " " + FLAGS_VALUE.getPreferredName() + " supports string or number");
+            }
+        }, FLAGS_VALUE, ObjectParser.ValueType.VALUE);
+        PARSER.declareStringOrNull(Builder::setFlags, FLAGS_VALUE);
+    }
+
+    private int flagsValue;
+    private int maxDeterminizedStates;
+
+    private RegexOptions() {
+    }
+
+    private RegexOptions(int flagsValue, int maxDeterminizedStates) {
+        this.flagsValue = flagsValue;
+        this.maxDeterminizedStates = maxDeterminizedStates;
+    }
+
+    /**
+     * Returns internal regular expression syntax flag value
+     * see {@link RegexpFlag#value()}
+     */
+    public int getFlagsValue() {
+        return flagsValue;
+    }
+
+    /**
+     * Returns the maximum automaton states allowed for fuzzy expansion
+     */
+    public int getMaxDeterminizedStates() {
+        return maxDeterminizedStates;
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    static RegexOptions parse(XContentParser parser) throws IOException {
+        return PARSER.parse(parser).build();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        RegexOptions that = (RegexOptions) o;
+
+        if (flagsValue != that.flagsValue) return false;
+        return maxDeterminizedStates == that.maxDeterminizedStates;
+
+    }
+
+    @Override
+    public int hashCode() {
+        int result = flagsValue;
+        result = 31 * result + maxDeterminizedStates;
+        return result;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(REGEX_OPTIONS.getPreferredName());
+        builder.field(FLAGS_VALUE.getPreferredName(), flagsValue);
+        builder.field(MAX_DETERMINIZED_STATES.getPreferredName(), maxDeterminizedStates);
+        builder.endObject();
+        return builder;
+    }
+
+    public static RegexOptions readRegexOptions(StreamInput in) throws IOException {
+        RegexOptions regexOptions = new RegexOptions();
+        regexOptions.readFrom(in);
+        return regexOptions;
+    }
+
+    @Override
+    public RegexOptions readFrom(StreamInput in) throws IOException {
+        this.flagsValue = in.readVInt();
+        this.maxDeterminizedStates = in.readVInt();
+        return this;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(flagsValue);
+        out.writeVInt(maxDeterminizedStates);
+    }
+
+    /**
+     * Options for regular expression queries
+     */
+    public static class Builder {
+        private int flagsValue = RegExp.ALL;
+        private int maxDeterminizedStates = Operations.DEFAULT_MAX_DETERMINIZED_STATES;
+
+        public Builder() {
+        }
+
+        /**
+         * Sets the regular expression syntax flags
+         * see {@link RegexpFlag}
+         */
+        public Builder setFlags(String flags) {
+            this.flagsValue = RegexpFlag.resolveValue(flags);
+            return this;
+        }
+
+        private Builder setFlagsValue(int flagsValue) {
+            this.flagsValue = flagsValue;
+            return this;
+        }
+
+        /**
+         * Sets the maximum automaton states allowed for the regular expression expansion
+         */
+        public Builder setMaxDeterminizedStates(int maxDeterminizedStates) {
+            if (maxDeterminizedStates < 0) {
+                throw new IllegalArgumentException("maxDeterminizedStates must not be negative");
+            }
+            this.maxDeterminizedStates = maxDeterminizedStates;
+            return this;
+        }
+
+        public RegexOptions build() {
+            return new RegexOptions(flagsValue, maxDeterminizedStates);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryContextMapping.java
index dffbb1aa80d..c9cb165aef7 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryContextMapping.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryContextMapping.java
@@ -36,6 +36,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * A {@link ContextMapping} that uses a simple string as a criteria
@@ -44,7 +45,7 @@ import java.util.Set;
  * {@link CategoryQueryContext} defines options for constructing
  * a unit of query context for this context type
  */
-public class CategoryContextMapping extends ContextMapping {
+public class CategoryContextMapping extends ContextMapping<CategoryQueryContext> {
 
     private static final String FIELD_FIELDNAME = "path";
 
@@ -137,6 +138,11 @@ public class CategoryContextMapping extends ContextMapping {
         return (values == null) ? Collections.<CharSequence>emptySet() : values;
     }
 
+    @Override
+    protected CategoryQueryContext prototype() {
+        return CategoryQueryContext.PROTOTYPE;
+    }
+
     /**
      * Parse a list of {@link CategoryQueryContext}
      * using <code>parser</code>. A QueryContexts accepts one of the following forms:
@@ -154,19 +160,13 @@ public class CategoryContextMapping extends ContextMapping {
      *  </ul>
      */
     @Override
-    public List<QueryContext> parseQueryContext(XContentParser parser) throws IOException, ElasticsearchParseException {
-        List<QueryContext> queryContexts = new ArrayList<>();
-        Token token = parser.nextToken();
-        if (token == Token.START_OBJECT || token == Token.VALUE_STRING) {
-            CategoryQueryContext parse = CategoryQueryContext.parse(parser);
-            queryContexts.add(new QueryContext(parse.getCategory().toString(), parse.getBoost(), parse.isPrefix()));
-        } else if (token == Token.START_ARRAY) {
-            while (parser.nextToken() != Token.END_ARRAY) {
-                CategoryQueryContext parse = CategoryQueryContext.parse(parser);
-                queryContexts.add(new QueryContext(parse.getCategory().toString(), parse.getBoost(), parse.isPrefix()));
-            }
-        }
-        return queryContexts;
+    public List<InternalQueryContext> toInternalQueryContexts(List<CategoryQueryContext> queryContexts) {
+        List<InternalQueryContext> internalInternalQueryContexts = new ArrayList<>(queryContexts.size());
+        internalInternalQueryContexts.addAll(
+            queryContexts.stream()
+                .map(queryContext -> new InternalQueryContext(queryContext.getCategory(), queryContext.getBoost(), queryContext.isPrefix()))
+                .collect(Collectors.toList()));
+        return internalInternalQueryContexts;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java
index c4931265776..a164faff8b1 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java
@@ -22,11 +22,11 @@ package org.elasticsearch.search.suggest.completion.context;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.Objects;
 
 import static org.elasticsearch.search.suggest.completion.context.CategoryContextMapping.CONTEXT_BOOST;
 import static org.elasticsearch.search.suggest.completion.context.CategoryContextMapping.CONTEXT_PREFIX;
@@ -35,12 +35,15 @@ import static org.elasticsearch.search.suggest.completion.context.CategoryContex
 /**
  * Defines the query context for {@link CategoryContextMapping}
  */
-public final class CategoryQueryContext implements ToXContent {
-    private final CharSequence category;
+public final class CategoryQueryContext implements QueryContext {
+    public static final String NAME = "category";
+    public static final CategoryQueryContext PROTOTYPE = new CategoryQueryContext("", 1, false);
+
+    private final String category;
     private final boolean isPrefix;
     private final int boost;
 
-    private CategoryQueryContext(CharSequence category, int boost, boolean isPrefix) {
+    private CategoryQueryContext(String category, int boost, boolean isPrefix) {
         this.category = category;
         this.boost = boost;
         this.isPrefix = isPrefix;
@@ -49,7 +52,7 @@ public final class CategoryQueryContext implements ToXContent {
     /**
      * Returns the category of the context
      */
-    public CharSequence getCategory() {
+    public String getCategory() {
         return category;
     }
 
@@ -71,54 +74,36 @@ public final class CategoryQueryContext implements ToXContent {
         return new Builder();
     }
 
-    public static class Builder {
-        private CharSequence category;
-        private boolean isPrefix = false;
-        private int boost = 1;
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
 
-        public Builder() {
-        }
+        CategoryQueryContext that = (CategoryQueryContext) o;
 
-        /**
-         * Sets the category of the context.
-         * This is a required field
-         */
-        public Builder setCategory(CharSequence context) {
-            this.category = context;
-            return this;
-        }
+        if (isPrefix != that.isPrefix) return false;
+        if (boost != that.boost) return false;
+        return category != null ? category.equals(that.category) : that.category == null;
 
-        /**
-         * Sets if the context should be treated as a prefix or not.
-         * Defaults to false
-         */
-        public Builder setPrefix(boolean prefix) {
-            this.isPrefix = prefix;
-            return this;
-        }
-
-        /**
-         * Sets the query-time boost of the context.
-         * Defaults to 1.
-         */
-        public Builder setBoost(int boost) {
-            this.boost = boost;
-            return this;
-        }
-
-        public CategoryQueryContext build() {
-            return new CategoryQueryContext(category, boost, isPrefix);
-        }
     }
 
-    private static ObjectParser<Builder, Void> CATEGORY_PARSER = new ObjectParser<>("category", null);
+    @Override
+    public int hashCode() {
+        int result = category != null ? category.hashCode() : 0;
+        result = 31 * result + (isPrefix ? 1 : 0);
+        result = 31 * result + boost;
+        return result;
+    }
+
+    private static ObjectParser<Builder, Void> CATEGORY_PARSER = new ObjectParser<>(NAME, null);
     static {
-        CATEGORY_PARSER.declareString(Builder::setCategory, new ParseField("context"));
-        CATEGORY_PARSER.declareInt(Builder::setBoost, new ParseField("boost"));
-        CATEGORY_PARSER.declareBoolean(Builder::setPrefix, new ParseField("prefix"));
+        CATEGORY_PARSER.declareString(Builder::setCategory, new ParseField(CONTEXT_VALUE));
+        CATEGORY_PARSER.declareInt(Builder::setBoost, new ParseField(CONTEXT_BOOST));
+        CATEGORY_PARSER.declareBoolean(Builder::setPrefix, new ParseField(CONTEXT_PREFIX));
     }
 
-    public static CategoryQueryContext parse(XContentParser parser) throws IOException {
+    @Override
+    public CategoryQueryContext fromXContext(XContentParser parser) throws IOException {
         XContentParser.Token token = parser.currentToken();
         Builder builder = builder();
         if (token == XContentParser.Token.START_OBJECT) {
@@ -140,4 +125,49 @@ public final class CategoryQueryContext implements ToXContent {
         builder.endObject();
         return builder;
     }
+
+    public static class Builder {
+        private String category;
+        private boolean isPrefix = false;
+        private int boost = 1;
+
+        public Builder() {
+        }
+
+        /**
+         * Sets the category of the category.
+         * This is a required field
+         */
+        public Builder setCategory(String category) {
+            Objects.requireNonNull(category, "category must not be null");
+            this.category = category;
+            return this;
+        }
+
+        /**
+         * Sets if the context should be treated as a prefix or not.
+         * Defaults to false
+         */
+        public Builder setPrefix(boolean prefix) {
+            this.isPrefix = prefix;
+            return this;
+        }
+
+        /**
+         * Sets the query-time boost of the context.
+         * Defaults to 1.
+         */
+        public Builder setBoost(int boost) {
+            if (boost <= 0) {
+                throw new IllegalArgumentException("boost must be greater than 0");
+            }
+            this.boost = boost;
+            return this;
+        }
+
+        public CategoryQueryContext build() {
+            Objects.requireNonNull(category, "category must not be null");
+            return new CategoryQueryContext(category, boost, isPrefix);
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java
index 42e5cc0a157..959a749a858 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java
@@ -23,11 +23,13 @@ import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
@@ -38,7 +40,7 @@ import java.util.Set;
  *
  * Implementations have to define how contexts are parsed at query/index time
  */
-public abstract class ContextMapping implements ToXContent {
+public abstract class ContextMapping<T extends QueryContext> implements ToXContent {
 
     public static final String FIELD_TYPE = "type";
     public static final String FIELD_NAME = "name";
@@ -94,10 +96,31 @@ public abstract class ContextMapping implements ToXContent {
      */
     protected abstract Set<CharSequence> parseContext(ParseContext.Document document);
 
+    /**
+     * Prototype for the query context
+     */
+    protected abstract T prototype();
+
     /**
      * Parses query contexts for this mapper
      */
-    public abstract List<QueryContext> parseQueryContext(XContentParser parser) throws IOException, ElasticsearchParseException;
+    public final List<InternalQueryContext> parseQueryContext(XContentParser parser) throws IOException, ElasticsearchParseException {
+        List<T> queryContexts = new ArrayList<>();
+        Token token = parser.nextToken();
+        if (token == Token.START_OBJECT || token == Token.VALUE_STRING) {
+            queryContexts.add((T) prototype().fromXContext(parser));
+        } else if (token == Token.START_ARRAY) {
+            while (parser.nextToken() != Token.END_ARRAY) {
+                queryContexts.add((T) prototype().fromXContext(parser));
+            }
+        }
+        return toInternalQueryContexts(queryContexts);
+    }
+
+    /**
+     * Convert query contexts to common representation
+     */
+    protected abstract List<InternalQueryContext> toInternalQueryContexts(List<T> queryContexts);
 
     /**
      * Implementations should add specific configurations
@@ -136,17 +159,38 @@ public abstract class ContextMapping implements ToXContent {
         }
     }
 
-    public static class QueryContext {
+    public static class InternalQueryContext {
         public final String context;
         public final int boost;
         public final boolean isPrefix;
 
-        public QueryContext(String context, int boost, boolean isPrefix) {
+        public InternalQueryContext(String context, int boost, boolean isPrefix) {
             this.context = context;
             this.boost = boost;
             this.isPrefix = isPrefix;
         }
 
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            InternalQueryContext that = (InternalQueryContext) o;
+
+            if (boost != that.boost) return false;
+            if (isPrefix != that.isPrefix) return false;
+            return context != null ? context.equals(that.context) : that.context == null;
+
+        }
+
+        @Override
+        public int hashCode() {
+            int result = context != null ? context.hashCode() : 0;
+            result = 31 * result + boost;
+            result = 31 * result + (isPrefix ? 1 : 0);
+            return result;
+        }
+
         @Override
         public String toString() {
             return "QueryContext{" +
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java
index 9d4bed4f664..ff550a8d34e 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java
@@ -43,7 +43,6 @@ import java.util.Set;
 
 import static org.elasticsearch.search.suggest.completion.context.ContextMapping.FIELD_NAME;
 import static org.elasticsearch.search.suggest.completion.context.ContextMapping.FIELD_TYPE;
-import static org.elasticsearch.search.suggest.completion.context.ContextMapping.QueryContext;
 import static org.elasticsearch.search.suggest.completion.context.ContextMapping.Type;
 
 /**
@@ -153,7 +152,7 @@ public class ContextMappings implements ToXContent {
      * @param queryContexts a map of context mapping name and collected query contexts
      * @return a context-enabled query
      */
-    public ContextQuery toContextQuery(CompletionQuery query, Map<String, List<QueryContext>> queryContexts) {
+    public ContextQuery toContextQuery(CompletionQuery query, Map<String, List<ContextMapping.InternalQueryContext>> queryContexts) {
         ContextQuery typedContextQuery = new ContextQuery(query);
         if (queryContexts.isEmpty() == false) {
             CharsRefBuilder scratch = new CharsRefBuilder();
@@ -162,9 +161,9 @@ public class ContextMappings implements ToXContent {
                 scratch.setCharAt(0, (char) typeId);
                 scratch.setLength(1);
                 ContextMapping mapping = contextMappings.get(typeId);
-                List<QueryContext> queryContext = queryContexts.get(mapping.name());
-                if (queryContext != null) {
-                    for (QueryContext context : queryContext) {
+                List<ContextMapping.InternalQueryContext> internalQueryContext = queryContexts.get(mapping.name());
+                if (internalQueryContext != null) {
+                    for (ContextMapping.InternalQueryContext context : internalQueryContext) {
                         scratch.append(context.context);
                         typedContextQuery.addContext(scratch.toCharsRef(), context.boost, !context.isPrefix);
                         scratch.setLength(1);
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java
index 4af90ab24a2..41d78e75353 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java
@@ -42,6 +42,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.apache.lucene.spatial.util.GeoHashUtils.addNeighbors;
 import static org.apache.lucene.spatial.util.GeoHashUtils.stringEncode;
@@ -56,7 +57,7 @@ import static org.apache.lucene.spatial.util.GeoHashUtils.stringEncode;
  * {@link GeoQueryContext} defines the options for constructing
  * a unit of query context for this context type
  */
-public class GeoContextMapping extends ContextMapping {
+public class GeoContextMapping extends ContextMapping<GeoQueryContext> {
 
     public static final String FIELD_PRECISION = "precision";
     public static final String FIELD_FIELDNAME = "path";
@@ -221,6 +222,11 @@ public class GeoContextMapping extends ContextMapping {
         return locations;
     }
 
+    @Override
+    protected GeoQueryContext prototype() {
+        return GeoQueryContext.PROTOTYPE;
+    }
+
     /**
      * Parse a list of {@link GeoQueryContext}
      * using <code>parser</code>. A QueryContexts accepts one of the following forms:
@@ -245,22 +251,10 @@ public class GeoContextMapping extends ContextMapping {
      * see {@link GeoUtils#parseGeoPoint(String, GeoPoint)} for GEO POINT
      */
     @Override
-    public List<QueryContext> parseQueryContext(XContentParser parser) throws IOException, ElasticsearchParseException {
-        List<GeoQueryContext> queryContexts = new ArrayList<>();
-        Token token = parser.nextToken();
-        if (token == Token.START_OBJECT || token == Token.VALUE_STRING) {
-            queryContexts.add(GeoQueryContext.parse(parser));
-        } else if (token == Token.START_ARRAY) {
-            while (parser.nextToken() != Token.END_ARRAY) {
-                queryContexts.add(GeoQueryContext.parse(parser));
-            }
-        }
-        List<QueryContext> queryContextList = new ArrayList<>();
+    public List<InternalQueryContext> toInternalQueryContexts(List<GeoQueryContext> queryContexts) {
+        List<InternalQueryContext> internalQueryContextList = new ArrayList<>();
         for (GeoQueryContext queryContext : queryContexts) {
-            int minPrecision = this.precision;
-            if (queryContext.getPrecision() != -1) {
-                minPrecision = Math.min(minPrecision, queryContext.getPrecision());
-            }
+            int minPrecision = Math.min(this.precision, queryContext.getPrecision());
             GeoPoint point = queryContext.getGeoPoint();
             final Collection<String> locations = new HashSet<>();
             String geoHash = stringEncode(point.getLon(), point.getLat(), minPrecision);
@@ -268,19 +262,20 @@ public class GeoContextMapping extends ContextMapping {
             if (queryContext.getNeighbours().isEmpty() && geoHash.length() == this.precision) {
                 addNeighbors(geoHash, locations);
             } else if (queryContext.getNeighbours().isEmpty() == false) {
-                for (Integer neighbourPrecision : queryContext.getNeighbours()) {
-                    if (neighbourPrecision < geoHash.length()) {
+                queryContext.getNeighbours().stream()
+                    .filter(neighbourPrecision -> neighbourPrecision < geoHash.length())
+                    .forEach(neighbourPrecision -> {
                         String truncatedGeoHash = geoHash.substring(0, neighbourPrecision);
                         locations.add(truncatedGeoHash);
                         addNeighbors(truncatedGeoHash, locations);
-                    }
-                }
-            }
-            for (String location : locations) {
-                queryContextList.add(new QueryContext(location, queryContext.getBoost(), location.length() < this.precision));
+                    });
             }
+            internalQueryContextList.addAll(
+                locations.stream()
+                    .map(location -> new InternalQueryContext(location, queryContext.getBoost(), location.length() < this.precision))
+                    .collect(Collectors.toList()));
         }
-        return queryContextList;
+        return internalQueryContextList;
     }
 
     @Override
@@ -304,7 +299,7 @@ public class GeoContextMapping extends ContextMapping {
         private int precision = DEFAULT_PRECISION;
         private String fieldName = null;
 
-        protected Builder(String name) {
+        public Builder(String name) {
             super(name);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java
index da9191bf2d5..913702c18d0 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java
@@ -24,13 +24,13 @@ import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 import static org.elasticsearch.search.suggest.completion.context.GeoContextMapping.CONTEXT_BOOST;
 import static org.elasticsearch.search.suggest.completion.context.GeoContextMapping.CONTEXT_NEIGHBOURS;
@@ -40,7 +40,10 @@ import static org.elasticsearch.search.suggest.completion.context.GeoContextMapp
 /**
  * Defines the query context for {@link GeoContextMapping}
  */
-public final class GeoQueryContext implements ToXContent {
+public final class GeoQueryContext implements QueryContext {
+    public static final String NAME = "geo";
+    public static final GeoQueryContext PROTOTYPE = new GeoQueryContext(null, 1, 12, Collections.emptyList());
+
     private final GeoPoint geoPoint;
     private final int boost;
     private final int precision;
@@ -81,90 +84,47 @@ public final class GeoQueryContext implements ToXContent {
         return neighbours;
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        GeoQueryContext that = (GeoQueryContext) o;
+
+        if (boost != that.boost) return false;
+        if (precision != that.precision) return false;
+        if (geoPoint != null ? !geoPoint.equals(that.geoPoint) : that.geoPoint != null) return false;
+        return neighbours != null ? neighbours.equals(that.neighbours) : that.neighbours == null;
+
+    }
+
+    @Override
+    public int hashCode() {
+        int result = geoPoint != null ? geoPoint.hashCode() : 0;
+        result = 31 * result + boost;
+        result = 31 * result + precision;
+        result = 31 * result + (neighbours != null ? neighbours.hashCode() : 0);
+        return result;
+    }
+
     public static Builder builder() {
         return new Builder();
     }
 
-    public static class Builder {
-        private GeoPoint geoPoint;
-        private int boost = 1;
-        private int precision = -1;
-        private List<Integer> neighbours = Collections.emptyList();
-
-        public Builder() {
-        }
-
-        /**
-         * Sets the query-time boost for the context
-         * Defaults to 1
-         */
-        public Builder setBoost(int boost) {
-            this.boost = boost;
-            return this;
-        }
-
-        /**
-         * Sets the precision level for computing the geohash from the context geo point.
-         * Defaults to using index-time precision level
-         */
-        public Builder setPrecision(int precision) {
-            this.precision = precision;
-            return this;
-        }
-
-        /**
-         * Sets the precision levels at which geohash cells neighbours are considered.
-         * Defaults to only considering neighbours at the index-time precision level
-         */
-        public Builder setNeighbours(List<Integer> neighbours) {
-            this.neighbours = neighbours;
-            return this;
-        }
-
-        /**
-         * Sets the geo point of the context.
-         * This is a required field
-         */
-        public Builder setGeoPoint(GeoPoint geoPoint) {
-            this.geoPoint = geoPoint;
-            return this;
-        }
-
-        private double lat = Double.NaN;
-        void setLat(double lat) {
-            this.lat = lat;
-        }
-
-        private double lon = Double.NaN;
-        void setLon(double lon) {
-            this.lon = lon;
-        }
-
-        public GeoQueryContext build() {
-            if (geoPoint == null) {
-                if (Double.isNaN(lat) == false && Double.isNaN(lon) == false) {
-                    geoPoint = new GeoPoint(lat, lon);
-                } else {
-                    throw new IllegalArgumentException("no geohash or geo point provided");
-                }
-            }
-            return new GeoQueryContext(geoPoint, boost, precision, neighbours);
-        }
-    }
-
-    private static ObjectParser<GeoQueryContext.Builder, Void> GEO_CONTEXT_PARSER = new ObjectParser<>("geo", null);
+    private static ObjectParser<GeoQueryContext.Builder, Void> GEO_CONTEXT_PARSER = new ObjectParser<>(NAME, null);
     static {
-        GEO_CONTEXT_PARSER.declareField((parser, geoQueryContext, geoContextMapping) -> geoQueryContext.setGeoPoint(GeoUtils.parseGeoPoint(parser)), new ParseField("context"), ObjectParser.ValueType.OBJECT);
-        GEO_CONTEXT_PARSER.declareInt(GeoQueryContext.Builder::setBoost, new ParseField("boost"));
+        GEO_CONTEXT_PARSER.declareField((parser, geoQueryContext, geoContextMapping) -> geoQueryContext.setGeoPoint(GeoUtils.parseGeoPoint(parser)), new ParseField(CONTEXT_VALUE), ObjectParser.ValueType.OBJECT);
+        GEO_CONTEXT_PARSER.declareInt(GeoQueryContext.Builder::setBoost, new ParseField(CONTEXT_BOOST));
         // TODO : add string support for precision for GeoUtils.geoHashLevelsForPrecision()
-        GEO_CONTEXT_PARSER.declareInt(GeoQueryContext.Builder::setPrecision, new ParseField("precision"));
+        GEO_CONTEXT_PARSER.declareInt(GeoQueryContext.Builder::setPrecision, new ParseField(CONTEXT_PRECISION));
         // TODO : add string array support for precision for GeoUtils.geoHashLevelsForPrecision()
-        GEO_CONTEXT_PARSER.declareIntArray(GeoQueryContext.Builder::setNeighbours, new ParseField("neighbours"));
+        GEO_CONTEXT_PARSER.declareIntArray(GeoQueryContext.Builder::setNeighbours, new ParseField(CONTEXT_NEIGHBOURS));
         GEO_CONTEXT_PARSER.declareDouble(GeoQueryContext.Builder::setLat, new ParseField("lat"));
         GEO_CONTEXT_PARSER.declareDouble(GeoQueryContext.Builder::setLon, new ParseField("lon"));
     }
 
-    public static GeoQueryContext parse(XContentParser parser) throws IOException {
+    @Override
+    public GeoQueryContext fromXContext(XContentParser parser) throws IOException {
         XContentParser.Token token = parser.currentToken();
         GeoQueryContext.Builder builder = new Builder();
         if (token == XContentParser.Token.START_OBJECT) {
@@ -190,4 +150,82 @@ public final class GeoQueryContext implements ToXContent {
         builder.endObject();
         return builder;
     }
+
+    public static class Builder {
+        private GeoPoint geoPoint;
+        private int boost = 1;
+        private int precision = 12;
+        private List<Integer> neighbours = Collections.emptyList();
+
+        public Builder() {
+        }
+
+        /**
+         * Sets the query-time boost for the context
+         * Defaults to 1
+         */
+        public Builder setBoost(int boost) {
+            if (boost <= 0) {
+                throw new IllegalArgumentException("boost must be greater than 0");
+            }
+            this.boost = boost;
+            return this;
+        }
+
+        /**
+         * Sets the precision level for computing the geohash from the context geo point.
+         * Defaults to using index-time precision level
+         */
+        public Builder setPrecision(int precision) {
+            if (precision < 1 || precision > 12) {
+                throw new IllegalArgumentException("precision must be between 1 and 12");
+            }
+            this.precision = precision;
+            return this;
+        }
+
+        /**
+         * Sets the precision levels at which geohash cells neighbours are considered.
+         * Defaults to only considering neighbours at the index-time precision level
+         */
+        public Builder setNeighbours(List<Integer> neighbours) {
+            for (int neighbour : neighbours) {
+                if (neighbour < 1 || neighbour > 12) {
+                    throw new IllegalArgumentException("neighbour value must be between 1 and 12");
+                }
+            }
+            this.neighbours = neighbours;
+            return this;
+        }
+
+        /**
+         * Sets the geo point of the context.
+         * This is a required field
+         */
+        public Builder setGeoPoint(GeoPoint geoPoint) {
+            Objects.requireNonNull(geoPoint, "geoPoint must not be null");
+            this.geoPoint = geoPoint;
+            return this;
+        }
+
+        private double lat = Double.NaN;
+        void setLat(double lat) {
+            this.lat = lat;
+        }
+
+        private double lon = Double.NaN;
+        void setLon(double lon) {
+            this.lon = lon;
+        }
+
+        public GeoQueryContext build() {
+            if (geoPoint == null) {
+                if (Double.isNaN(lat) == false && Double.isNaN(lon) == false) {
+                    geoPoint = new GeoPoint(lat, lon);
+                }
+            }
+            Objects.requireNonNull(geoPoint, "geoPoint must not be null");
+            return new GeoQueryContext(geoPoint, boost, precision, neighbours);
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortParser.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/QueryContext.java
similarity index 71%
rename from core/src/main/java/org/elasticsearch/search/sort/SortParser.java
rename to core/src/main/java/org/elasticsearch/search/suggest/completion/context/QueryContext.java
index 6383afd8845..9d96bf81447 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/SortParser.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/QueryContext.java
@@ -17,18 +17,17 @@
  * under the License.
  */
 
-package org.elasticsearch.search.sort;
+package org.elasticsearch.search.suggest.completion.context;
 
-import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
 
 /**
- *
+ * Interface for serializing/de-serializing completion query context
  */
-public interface SortParser {
+public interface QueryContext extends ToXContent {
 
-    String[] names();
-
-    SortField parse(XContentParser parser, SearchContext context) throws Exception;
+    QueryContext fromXContext(XContentParser parser) throws IOException;
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGenerator.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGenerator.java
index 5b937500d6b..a454735ae1c 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGenerator.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGenerator.java
@@ -178,7 +178,7 @@ public final class DirectCandidateGenerator extends CandidateGenerator {
 
     protected long thresholdFrequency(long termFrequency, long dictionarySize) {
         if (termFrequency > 0) {
-            return (long) Math.max(0, Math.round(termFrequency * (Math.log10(termFrequency - frequencyPlateau) * (1.0 / Math.log10(logBase))) + 1));
+            return Math.max(0, Math.round(termFrequency * (Math.log10(termFrequency - frequencyPlateau) * (1.0 / Math.log10(logBase))) + 1));
         }
         return 0;
 
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
index 90ec2845b8a..f6a94314f5b 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
@@ -25,13 +25,12 @@ import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.suggest.SortBy;
 import org.elasticsearch.search.suggest.SuggestUtils;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.CandidateGenerator;
 
@@ -42,7 +41,7 @@ import java.util.Set;
 import java.util.function.Consumer;
 
 public final class DirectCandidateGeneratorBuilder
-        implements Writeable<DirectCandidateGeneratorBuilder>, CandidateGenerator {
+        implements CandidateGenerator {
 
     private static final String TYPE = "direct_generator";
     static final DirectCandidateGeneratorBuilder PROTOTYPE = new DirectCandidateGeneratorBuilder("_na_");
@@ -350,8 +349,7 @@ public final class DirectCandidateGeneratorBuilder
         return replaceField(tmpFieldName.iterator().next(), tempGenerator);
     }
 
-    public PhraseSuggestionContext.DirectCandidateGenerator build(QueryShardContext context) throws IOException {
-        MapperService mapperService = context.getMapperService();
+    public PhraseSuggestionContext.DirectCandidateGenerator build(MapperService mapperService) throws IOException {
         PhraseSuggestionContext.DirectCandidateGenerator generator = new PhraseSuggestionContext.DirectCandidateGenerator();
         generator.setField(this.field);
         transferIfNotNull(this.size, generator::size);
@@ -372,7 +370,7 @@ public final class DirectCandidateGeneratorBuilder
             generator.suggestMode(SuggestUtils.resolveSuggestMode(this.suggestMode));
         }
         if (this.sort != null) {
-            generator.sort(SuggestUtils.resolveSort(this.sort));
+            generator.sort(SortBy.resolve(this.sort));
         }
         if (this.stringDistance != null) {
             generator.stringDistance(SuggestUtils.resolveDistance(this.stringDistance));
@@ -384,7 +382,7 @@ public final class DirectCandidateGeneratorBuilder
         transferIfNotNull(this.maxInspections, generator::maxInspections);
         transferIfNotNull(this.maxTermFreq, generator::maxTermFreq);
         transferIfNotNull(this.prefixLength, generator::prefixLength);
-        transferIfNotNull(this.minWordLength, generator::minQueryLength);
+        transferIfNotNull(this.minWordLength, generator::minWordLength);
         transferIfNotNull(this.minDocFreq, generator::minDocFreq);
         return generator;
     }
@@ -490,4 +488,4 @@ public final class DirectCandidateGeneratorBuilder
                 Objects.equals(minWordLength, other.minWordLength) &&
                 Objects.equals(minDocFreq, other.minDocFreq);
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java
new file mode 100644
index 00000000000..e11a920f966
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.phrase;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * An <a href="http://en.wikipedia.org/wiki/Additive_smoothing">additive
+ * smoothing</a> model.
+ * <p>
+ * See <a
+ * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
+ * Smoothing</a> for details.
+ * </p>
+ */
+public final class Laplace extends SmoothingModel {
+    private double alpha = DEFAULT_LAPLACE_ALPHA;
+    private static final String NAME = "laplace";
+    private static final ParseField ALPHA_FIELD = new ParseField("alpha");
+    static final ParseField PARSE_FIELD = new ParseField(NAME);
+    /**
+     * Default alpha parameter for laplace smoothing
+     */
+    public static final double DEFAULT_LAPLACE_ALPHA = 0.5;
+    public static final Laplace PROTOTYPE = new Laplace(DEFAULT_LAPLACE_ALPHA);
+
+    /**
+     * Creates a Laplace smoothing model.
+     *
+     */
+    public Laplace(double alpha) {
+        this.alpha = alpha;
+    }
+
+    /**
+     * @return the laplace model alpha parameter
+     */
+    public double getAlpha() {
+        return this.alpha;
+    }
+
+    @Override
+    protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(ALPHA_FIELD.getPreferredName(), alpha);
+        return builder;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeDouble(alpha);
+    }
+
+    @Override
+    public SmoothingModel readFrom(StreamInput in) throws IOException {
+        return new Laplace(in.readDouble());
+    }
+
+    @Override
+    protected boolean doEquals(SmoothingModel other) {
+        Laplace otherModel = (Laplace) other;
+        return Objects.equals(alpha, otherModel.alpha);
+    }
+
+    @Override
+    protected final int doHashCode() {
+        return Objects.hash(alpha);
+    }
+
+    @Override
+    public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
+        XContentParser parser = parseContext.parser();
+        XContentParser.Token token;
+        String fieldName = null;
+        double alpha = DEFAULT_LAPLACE_ALPHA;
+        while ((token = parser.nextToken()) != Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                fieldName = parser.currentName();
+            }
+            if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, ALPHA_FIELD)) {
+                alpha = parser.doubleValue();
+            }
+        }
+        return new Laplace(alpha);
+    }
+
+    @Override
+    public WordScorerFactory buildWordScorerFactory() {
+        return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
+                -> new LaplaceScorer(reader, terms,  field, realWordLikelyhood, separator, alpha);
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java
new file mode 100644
index 00000000000..b94ea333fdb
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.phrase;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Linear interpolation smoothing model.
+ * <p>
+ * See <a
+ * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
+ * Smoothing</a> for details.
+ * </p>
+ */
+public final class LinearInterpolation extends SmoothingModel {
+    private static final String NAME = "linear";
+    public static final LinearInterpolation PROTOTYPE = new LinearInterpolation(0.8, 0.1, 0.1);
+    private final double trigramLambda;
+    private final double bigramLambda;
+    private final double unigramLambda;
+    static final ParseField PARSE_FIELD = new ParseField(NAME);
+    private static final ParseField TRIGRAM_FIELD = new ParseField("trigram_lambda");
+    private static final ParseField BIGRAM_FIELD = new ParseField("bigram_lambda");
+    private static final ParseField UNIGRAM_FIELD = new ParseField("unigram_lambda");
+
+    /**
+     * Creates a linear interpolation smoothing model.
+     *
+     * Note: the lambdas must sum up to one.
+     *
+     * @param trigramLambda
+     *            the trigram lambda
+     * @param bigramLambda
+     *            the bigram lambda
+     * @param unigramLambda
+     *            the unigram lambda
+     */
+    public LinearInterpolation(double trigramLambda, double bigramLambda, double unigramLambda) {
+        double sum = trigramLambda + bigramLambda + unigramLambda;
+        if (Math.abs(sum - 1.0) > 0.001) {
+            throw new IllegalArgumentException("linear smoothing lambdas must sum to 1");
+        }
+        this.trigramLambda = trigramLambda;
+        this.bigramLambda = bigramLambda;
+        this.unigramLambda = unigramLambda;
+    }
+
+    public double getTrigramLambda() {
+        return this.trigramLambda;
+    }
+
+    public double getBigramLambda() {
+        return this.bigramLambda;
+    }
+
+    public double getUnigramLambda() {
+        return this.unigramLambda;
+    }
+
+    @Override
+    protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(TRIGRAM_FIELD.getPreferredName(), trigramLambda);
+        builder.field(BIGRAM_FIELD.getPreferredName(), bigramLambda);
+        builder.field(UNIGRAM_FIELD.getPreferredName(), unigramLambda);
+        return builder;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeDouble(trigramLambda);
+        out.writeDouble(bigramLambda);
+        out.writeDouble(unigramLambda);
+    }
+
+    @Override
+    public LinearInterpolation readFrom(StreamInput in) throws IOException {
+        return new LinearInterpolation(in.readDouble(), in.readDouble(), in.readDouble());
+    }
+
+    @Override
+    protected boolean doEquals(SmoothingModel other) {
+        final LinearInterpolation otherModel = (LinearInterpolation) other;
+        return Objects.equals(trigramLambda, otherModel.trigramLambda) &&
+                Objects.equals(bigramLambda, otherModel.bigramLambda) &&
+                Objects.equals(unigramLambda, otherModel.unigramLambda);
+    }
+
+    @Override
+    protected final int doHashCode() {
+        return Objects.hash(trigramLambda, bigramLambda, unigramLambda);
+    }
+
+    @Override
+    public LinearInterpolation innerFromXContent(QueryParseContext parseContext) throws IOException {
+        XContentParser parser = parseContext.parser();
+        XContentParser.Token token;
+        String fieldName = null;
+        double trigramLambda = 0.0;
+        double bigramLambda = 0.0;
+        double unigramLambda = 0.0;
+        ParseFieldMatcher matcher = parseContext.parseFieldMatcher();
+        while ((token = parser.nextToken()) != Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                fieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (matcher.match(fieldName, TRIGRAM_FIELD)) {
+                    trigramLambda = parser.doubleValue();
+                    if (trigramLambda < 0) {
+                        throw new IllegalArgumentException("trigram_lambda must be positive");
+                    }
+                } else if (matcher.match(fieldName, BIGRAM_FIELD)) {
+                    bigramLambda = parser.doubleValue();
+                    if (bigramLambda < 0) {
+                        throw new IllegalArgumentException("bigram_lambda must be positive");
+                    }
+                } else if (matcher.match(fieldName, UNIGRAM_FIELD)) {
+                    unigramLambda = parser.doubleValue();
+                    if (unigramLambda < 0) {
+                        throw new IllegalArgumentException("unigram_lambda must be positive");
+                    }
+                } else {
+                    throw new IllegalArgumentException(
+                            "suggester[phrase][smoothing][linear] doesn't support field [" + fieldName + "]");
+                }
+            } else {
+                throw new ParsingException(parser.getTokenLocation(),
+                        "[" + NAME + "] unknown token [" + token + "] after [" + fieldName + "]");
+            }
+        }
+        return new LinearInterpolation(trigramLambda, bigramLambda, unigramLambda);
+    }
+
+    @Override
+    public WordScorerFactory buildWordScorerFactory() {
+        return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) ->
+                    new LinearInterpoatingScorer(reader, terms, field, realWordLikelyhood, separator, trigramLambda, bigramLambda,
+                        unigramLambda);
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java
deleted file mode 100644
index fc60fc6fc80..00000000000
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java
+++ /dev/null
@@ -1,358 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.suggest.phrase;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
-import org.elasticsearch.index.fielddata.IndexFieldDataService;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.script.CompiledScript;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.Template;
-import org.elasticsearch.search.suggest.SuggestContextParser;
-import org.elasticsearch.search.suggest.SuggestUtils;
-import org.elasticsearch.search.suggest.SuggestionSearchContext;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
-
-import java.io.IOException;
-import java.util.Collections;
-
-public final class PhraseSuggestParser implements SuggestContextParser {
-
-    private PhraseSuggester suggester;
-
-    public PhraseSuggestParser(PhraseSuggester suggester) {
-        this.suggester = suggester;
-    }
-
-    @Override
-    public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException {
-        PhraseSuggestionContext suggestion = new PhraseSuggestionContext(suggester);
-        ParseFieldMatcher parseFieldMatcher = mapperService.getIndexSettings().getParseFieldMatcher();
-        XContentParser.Token token;
-        String fieldName = null;
-        boolean gramSizeSet = false;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                fieldName = parser.currentName();
-            } else if (token.isValue()) {
-                if (!SuggestUtils.parseSuggestContext(parser, mapperService, fieldName, suggestion, parseFieldMatcher)) {
-                    if ("real_word_error_likelihood".equals(fieldName) || "realWorldErrorLikelihood".equals(fieldName)) {
-                        suggestion.setRealWordErrorLikelihood(parser.floatValue());
-                        if (suggestion.realworldErrorLikelyhood() <= 0.0) {
-                            throw new IllegalArgumentException("real_word_error_likelihood must be > 0.0");
-                        }
-                    } else if ("confidence".equals(fieldName)) {
-                        suggestion.setConfidence(parser.floatValue());
-                        if (suggestion.confidence() < 0.0) {
-                            throw new IllegalArgumentException("confidence must be >= 0.0");
-                        }
-                    } else if ("separator".equals(fieldName)) {
-                        suggestion.setSeparator(new BytesRef(parser.text()));
-                    } else if ("max_errors".equals(fieldName) || "maxErrors".equals(fieldName)) {
-                        suggestion.setMaxErrors(parser.floatValue());
-                        if (suggestion.maxErrors() <= 0.0) {
-                            throw new IllegalArgumentException("max_error must be > 0.0");
-                        }
-                    } else if ("gram_size".equals(fieldName) || "gramSize".equals(fieldName)) {
-                        suggestion.setGramSize(parser.intValue());
-                        if (suggestion.gramSize() < 1) {
-                            throw new IllegalArgumentException("gram_size must be >= 1");
-                        }
-                        gramSizeSet = true;
-                    } else if ("force_unigrams".equals(fieldName) || "forceUnigrams".equals(fieldName)) {
-                        suggestion.setRequireUnigram(parser.booleanValue());
-                    } else if ("token_limit".equals(fieldName) || "tokenLimit".equals(fieldName)) {
-                        int tokenLimit = parser.intValue();
-                        if (tokenLimit <= 0) {
-                            throw new IllegalArgumentException("token_limit must be >= 1");
-                        }
-                        suggestion.setTokenLimit(tokenLimit);
-                    } else {
-                        throw new IllegalArgumentException("suggester[phrase] doesn't support field [" + fieldName + "]");
-                    }
-                }
-            } else if (token == Token.START_ARRAY) {
-                if (parseFieldMatcher.match(fieldName, DirectCandidateGeneratorBuilder.DIRECT_GENERATOR_FIELD)) {
-                    // for now we only have a single type of generators
-                    while ((token = parser.nextToken()) == Token.START_OBJECT) {
-                        PhraseSuggestionContext.DirectCandidateGenerator generator = parseCandidateGenerator(parser, mapperService, parseFieldMatcher);
-                        verifyGenerator(generator);
-                        suggestion.addGenerator(generator);
-                    }
-                } else {
-                    throw new IllegalArgumentException("suggester[phrase]  doesn't support array field [" + fieldName + "]");
-                }
-            } else if (token == Token.START_OBJECT) {
-                if ("smoothing".equals(fieldName)) {
-                    parseSmoothingModel(parser, suggestion, fieldName);
-                } else if ("highlight".equals(fieldName)) {
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            fieldName = parser.currentName();
-                        } else if (token.isValue()) {
-                            if ("pre_tag".equals(fieldName) || "preTag".equals(fieldName)) {
-                                suggestion.setPreTag(parser.utf8Bytes());
-                            } else if ("post_tag".equals(fieldName) || "postTag".equals(fieldName)) {
-                                suggestion.setPostTag(parser.utf8Bytes());
-                            } else {
-                                throw new IllegalArgumentException(
-                                    "suggester[phrase][highlight] doesn't support field [" + fieldName + "]");
-                            }
-                        }
-                    }
-                } else if ("collate".equals(fieldName)) {
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            fieldName = parser.currentName();
-                        } else if ("query".equals(fieldName)) {
-                            if (suggestion.getCollateQueryScript() != null) {
-                                throw new IllegalArgumentException("suggester[phrase][collate] query already set, doesn't support additional [" + fieldName + "]");
-                            }
-                            Template template = Template.parse(parser, parseFieldMatcher);
-                            CompiledScript compiledScript = suggester.scriptService().compile(template, ScriptContext.Standard.SEARCH, Collections.emptyMap());
-                            suggestion.setCollateQueryScript(compiledScript);
-                        } else if ("params".equals(fieldName)) {
-                            suggestion.setCollateScriptParams(parser.map());
-                        } else if ("prune".equals(fieldName)) {
-                            if (parser.isBooleanValue()) {
-                                suggestion.setCollatePrune(parser.booleanValue());
-                            } else {
-                                throw new IllegalArgumentException("suggester[phrase][collate] prune must be either 'true' or 'false'");
-                            }
-                        } else {
-                            throw new IllegalArgumentException(
-                                    "suggester[phrase][collate] doesn't support field [" + fieldName + "]");
-                        }
-                    }
-                } else {
-                    throw new IllegalArgumentException("suggester[phrase]  doesn't support array field [" + fieldName + "]");
-                }
-            } else {
-                throw new IllegalArgumentException("suggester[phrase] doesn't support field [" + fieldName + "]");
-            }
-        }
-
-        if (suggestion.getField() == null) {
-            throw new IllegalArgumentException("The required field option is missing");
-        }
-
-        MappedFieldType fieldType = mapperService.fullName(suggestion.getField());
-        if (fieldType == null) {
-            throw new IllegalArgumentException("No mapping found for field [" + suggestion.getField() + "]");
-        } else if (suggestion.getAnalyzer() == null) {
-            // no analyzer name passed in, so try the field's analyzer, or the default analyzer
-            if (fieldType.searchAnalyzer() == null) {
-                suggestion.setAnalyzer(mapperService.searchAnalyzer());
-            } else {
-                suggestion.setAnalyzer(fieldType.searchAnalyzer());
-            }
-        }
-
-        if (suggestion.model() == null) {
-            suggestion.setModel(StupidBackoffScorer.FACTORY);
-        }
-
-        if (!gramSizeSet || suggestion.generators().isEmpty()) {
-            final ShingleTokenFilterFactory.Factory shingleFilterFactory = SuggestUtils.getShingleFilterFactory(suggestion.getAnalyzer());
-            if (!gramSizeSet) {
-                // try to detect the shingle size
-                if (shingleFilterFactory != null) {
-                    suggestion.setGramSize(shingleFilterFactory.getMaxShingleSize());
-                    if (suggestion.getAnalyzer() == null && shingleFilterFactory.getMinShingleSize() > 1 && !shingleFilterFactory.getOutputUnigrams()) {
-                        throw new IllegalArgumentException("The default analyzer for field: [" + suggestion.getField() + "] doesn't emit unigrams. If this is intentional try to set the analyzer explicitly");
-                    }
-                }
-            }
-            if (suggestion.generators().isEmpty()) {
-                if (shingleFilterFactory != null && shingleFilterFactory.getMinShingleSize() > 1 && !shingleFilterFactory.getOutputUnigrams() && suggestion.getRequireUnigram()) {
-                    throw new IllegalArgumentException("The default candidate generator for phrase suggest can't operate on field: [" + suggestion.getField() + "] since it doesn't emit unigrams. If this is intentional try to set the candidate generator field explicitly");
-                }
-                // use a default generator on the same field
-                DirectCandidateGenerator generator = new DirectCandidateGenerator();
-                generator.setField(suggestion.getField());
-                suggestion.addGenerator(generator);
-            }
-        }
-
-
-
-        return suggestion;
-    }
-
-    public void parseSmoothingModel(XContentParser parser, PhraseSuggestionContext suggestion, String fieldName) throws IOException {
-        XContentParser.Token token;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                fieldName = parser.currentName();
-                if ("linear".equals(fieldName)) {
-                    ensureNoSmoothing(suggestion);
-                    final double[] lambdas = new double[3];
-                    while ((token = parser.nextToken()) != Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            fieldName = parser.currentName();
-                        }
-                        if (token.isValue()) {
-                            if ("trigram_lambda".equals(fieldName) || "trigramLambda".equals(fieldName)) {
-                                lambdas[0] = parser.doubleValue();
-                                if (lambdas[0] < 0) {
-                                    throw new IllegalArgumentException("trigram_lambda must be positive");
-                                }
-                            } else if ("bigram_lambda".equals(fieldName) || "bigramLambda".equals(fieldName)) {
-                                lambdas[1] = parser.doubleValue();
-                                if (lambdas[1] < 0) {
-                                    throw new IllegalArgumentException("bigram_lambda must be positive");
-                                }
-                            } else if ("unigram_lambda".equals(fieldName) || "unigramLambda".equals(fieldName)) {
-                                lambdas[2] = parser.doubleValue();
-                                if (lambdas[2] < 0) {
-                                    throw new IllegalArgumentException("unigram_lambda must be positive");
-                                }
-                            } else {
-                                throw new IllegalArgumentException(
-                                        "suggester[phrase][smoothing][linear] doesn't support field [" + fieldName + "]");
-                            }
-                        }
-                    }
-                    double sum = 0.0d;
-                    for (int i = 0; i < lambdas.length; i++) {
-                        sum += lambdas[i];
-                    }
-                    if (Math.abs(sum - 1.0) > 0.001) {
-                        throw new IllegalArgumentException("linear smoothing lambdas must sum to 1");
-                    }
-                    suggestion.setModel(new WordScorer.WordScorerFactory() {
-                        @Override
-                        public WordScorer newScorer(IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
-                                throws IOException {
-                            return new LinearInterpoatingScorer(reader, terms, field, realWordLikelyhood, separator, lambdas[0], lambdas[1],
-                                    lambdas[2]);
-                        }
-                    });
-                } else if ("laplace".equals(fieldName)) {
-                    ensureNoSmoothing(suggestion);
-                    double theAlpha = Laplace.DEFAULT_LAPLACE_ALPHA;
-
-                    while ((token = parser.nextToken()) != Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            fieldName = parser.currentName();
-                        }
-                        if (token.isValue() && "alpha".equals(fieldName)) {
-                            theAlpha = parser.doubleValue();
-                        }
-                    }
-                    final double alpha = theAlpha;
-                    suggestion.setModel(new WordScorer.WordScorerFactory() {
-                        @Override
-                        public WordScorer newScorer(IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
-                                throws IOException {
-                            return new LaplaceScorer(reader, terms,  field, realWordLikelyhood, separator, alpha);
-                        }
-                    });
-
-                } else if ("stupid_backoff".equals(fieldName) || "stupidBackoff".equals(fieldName)) {
-                    ensureNoSmoothing(suggestion);
-                    double theDiscount = StupidBackoff.DEFAULT_BACKOFF_DISCOUNT;
-                    while ((token = parser.nextToken()) != Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            fieldName = parser.currentName();
-                        }
-                        if (token.isValue() && "discount".equals(fieldName)) {
-                            theDiscount = parser.doubleValue();
-                        }
-                    }
-                    final double discount = theDiscount;
-                    suggestion.setModel(new WordScorer.WordScorerFactory() {
-                        @Override
-                        public WordScorer newScorer(IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
-                                throws IOException {
-                            return new StupidBackoffScorer(reader, terms, field, realWordLikelyhood, separator, discount);
-                        }
-                    });
-
-                } else {
-                    throw new IllegalArgumentException("suggester[phrase] doesn't support object field [" + fieldName + "]");
-                }
-            }
-        }
-    }
-
-    private void ensureNoSmoothing(PhraseSuggestionContext suggestion) {
-        if (suggestion.model() != null) {
-            throw new IllegalArgumentException("only one smoothing model supported");
-        }
-    }
-
-    private void verifyGenerator(PhraseSuggestionContext.DirectCandidateGenerator suggestion) {
-        // Verify options and set defaults
-        if (suggestion.field() == null) {
-            throw new IllegalArgumentException("The required field option is missing");
-        }
-    }
-
-    static PhraseSuggestionContext.DirectCandidateGenerator parseCandidateGenerator(XContentParser parser, MapperService mapperService,
-             ParseFieldMatcher parseFieldMatcher) throws IOException {
-        XContentParser.Token token;
-        String fieldName = null;
-        PhraseSuggestionContext.DirectCandidateGenerator generator = new PhraseSuggestionContext.DirectCandidateGenerator();
-        while ((token = parser.nextToken()) != Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                fieldName = parser.currentName();
-            }
-            if (token.isValue()) {
-                if (!SuggestUtils.parseDirectSpellcheckerSettings(parser, fieldName, generator, parseFieldMatcher)) {
-                    if ("field".equals(fieldName)) {
-                        generator.setField(parser.text());
-                        if (mapperService.fullName(generator.field()) == null) {
-                            throw new IllegalArgumentException("No mapping found for field [" + generator.field() + "]");
-                        }
-                    } else if ("size".equals(fieldName)) {
-                        generator.size(parser.intValue());
-                    } else if ("pre_filter".equals(fieldName) || "preFilter".equals(fieldName)) {
-                        String analyzerName = parser.text();
-                        Analyzer analyzer = mapperService.analysisService().analyzer(analyzerName);
-                        if (analyzer == null) {
-                            throw new IllegalArgumentException("Analyzer [" + analyzerName + "] doesn't exists");
-                        }
-                        generator.preFilter(analyzer);
-                    } else if ("post_filter".equals(fieldName) || "postFilter".equals(fieldName)) {
-                        String analyzerName = parser.text();
-                        Analyzer analyzer = mapperService.analysisService().analyzer(analyzerName);
-                        if (analyzer == null) {
-                            throw new IllegalArgumentException("Analyzer [" + analyzerName + "] doesn't exists");
-                        }
-                        generator.postFilter(analyzer);
-                    } else {
-                        throw new IllegalArgumentException("CandidateGenerator doesn't support [" + fieldName + "]");
-                    }
-                }
-            }
-        }
-        return generator;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
index 7838eacd960..14bced639f2 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
@@ -31,18 +31,17 @@ import org.apache.lucene.util.CharsRefBuilder;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.text.Text;
-import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.query.ParsedQuery;
-import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.suggest.Suggest.Suggestion;
 import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry;
 import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
-import org.elasticsearch.search.suggest.SuggestContextParser;
 import org.elasticsearch.search.suggest.SuggestUtils;
 import org.elasticsearch.search.suggest.Suggester;
+import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 import org.elasticsearch.search.suggest.phrase.NoisyChannelSpellChecker.Result;
 
 import java.io.IOException;
@@ -53,13 +52,8 @@ import java.util.Map;
 public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
     private final BytesRef SEPARATOR = new BytesRef(" ");
     private static final String SUGGESTION_TEMPLATE_VAR_NAME = "suggestion";
-    private final ScriptService scriptService;
-    private final IndicesService indicesService;
 
-    public PhraseSuggester(ScriptService scriptService, IndicesService indicesService) {
-        this.scriptService = scriptService;
-        this.indicesService = indicesService;
-    }
+    public static final PhraseSuggester PROTOTYPE = new PhraseSuggester();
 
     /*
      * More Ideas:
@@ -70,8 +64,8 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
      *   - phonetic filters could be interesting here too for candidate selection
      */
     @Override
-    public Suggestion<? extends Entry<? extends Option>> innerExecute(String name, PhraseSuggestionContext suggestion, IndexSearcher searcher,
-            CharsRefBuilder spare) throws IOException {
+    public Suggestion<? extends Entry<? extends Option>> innerExecute(String name, PhraseSuggestionContext suggestion,
+            IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
         double realWordErrorLikelihood = suggestion.realworldErrorLikelyhood();
         final PhraseSuggestion response = new PhraseSuggestion(name, suggestion.getSize());
         final IndexReader indexReader = searcher.getIndexReader();
@@ -90,14 +84,16 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
         final String suggestField = suggestion.getField();
         final Terms suggestTerms = MultiFields.getTerms(indexReader, suggestField);
         if (gens.size() > 0 && suggestTerms != null) {
-            final NoisyChannelSpellChecker checker = new NoisyChannelSpellChecker(realWordErrorLikelihood, suggestion.getRequireUnigram(), suggestion.getTokenLimit());
+            final NoisyChannelSpellChecker checker = new NoisyChannelSpellChecker(realWordErrorLikelihood, suggestion.getRequireUnigram(),
+                    suggestion.getTokenLimit());
             final BytesRef separator = suggestion.separator();
-            WordScorer wordScorer = suggestion.model().newScorer(indexReader, suggestTerms, suggestField, realWordErrorLikelihood, separator);
+            WordScorer wordScorer = suggestion.model().newScorer(indexReader, suggestTerms, suggestField, realWordErrorLikelihood,
+                    separator);
             Result checkerResult;
             try (TokenStream stream = checker.tokenStream(suggestion.getAnalyzer(), suggestion.getText(), spare, suggestion.getField())) {
-                checkerResult = checker.getCorrections(stream, new MultiCandidateGeneratorWrapper(suggestion.getShardSize(),
-                                                                                                         gens.toArray(new CandidateGenerator[gens.size()])), suggestion.maxErrors(),
-                                                              suggestion.getShardSize(), wordScorer, suggestion.confidence(), suggestion.gramSize());
+                checkerResult = checker.getCorrections(stream,
+                        new MultiCandidateGeneratorWrapper(suggestion.getShardSize(), gens.toArray(new CandidateGenerator[gens.size()])),
+                        suggestion.maxErrors(), suggestion.getShardSize(), wordScorer, suggestion.confidence(), suggestion.gramSize());
                 }
 
             PhraseSuggestion.Entry resultEntry = buildResultEntry(suggestion, spare, checkerResult.cutoffScore);
@@ -115,10 +111,10 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
                     // from the index for a correction, collateMatch is updated
                     final Map<String, Object> vars = suggestion.getCollateScriptParams();
                     vars.put(SUGGESTION_TEMPLATE_VAR_NAME, spare.toString());
+                    ScriptService scriptService = suggestion.getShardContext().getScriptService();
                     final ExecutableScript executable = scriptService.executable(collateScript, vars);
                     final BytesReference querySource = (BytesReference) executable.run();
-                    IndexService indexService = indicesService.indexService(suggestion.getIndex());
-                    final ParsedQuery parsedQuery = indexService.newQueryShardContext().parse(querySource);
+                    final ParsedQuery parsedQuery = suggestion.getShardContext().parse(querySource);
                     collateMatch = Lucene.exists(searcher, parsedQuery.query());
                 }
                 if (!collateMatch && !collatePrune) {
@@ -142,18 +138,14 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
         return response;
     }
 
-    private PhraseSuggestion.Entry buildResultEntry(PhraseSuggestionContext suggestion, CharsRefBuilder spare, double cutoffScore) {
+    private PhraseSuggestion.Entry buildResultEntry(SuggestionContext suggestion, CharsRefBuilder spare, double cutoffScore) {
         spare.copyUTF8Bytes(suggestion.getText());
         return new PhraseSuggestion.Entry(new Text(spare.toString()), 0, spare.length(), cutoffScore);
     }
 
-    ScriptService scriptService() {
-        return scriptService;
-    }
-
     @Override
-    public SuggestContextParser getContextParser() {
-        return new PhraseSuggestParser(this);
+    public SuggestionBuilder<?> getBuilderPrototype() {
+        return PhraseSuggestionBuilder.PROTOTYPE;
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
index 09b4012d5b7..a4793dfbdaa 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
@@ -18,26 +18,34 @@
  */
 package org.elasticsearch.search.suggest.phrase;
 
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.util.BytesRef;
+
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.Template;
-import org.elasticsearch.search.suggest.SuggestBuilder.SuggestionBuilder;
-import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
+import org.elasticsearch.search.suggest.SuggestUtils;
+import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -48,24 +56,69 @@ import java.util.Set;
 /**
  * Defines the actual suggest command for phrase suggestions ( <tt>phrase</tt>).
  */
-public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSuggestionBuilder> {
-    private Float maxErrors;
-    private String separator;
-    private Float realWordErrorLikelihood;
-    private Float confidence;
-    private final Map<String, List<CandidateGenerator>> generators = new HashMap<>();
+public class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSuggestionBuilder> {
+
+    private static final String SUGGESTION_NAME = "phrase";
+
+    public static final PhraseSuggestionBuilder PROTOTYPE = new PhraseSuggestionBuilder("_na_");
+
+    protected static final ParseField MAXERRORS_FIELD = new ParseField("max_errors");
+    protected static final ParseField RWE_LIKELIHOOD_FIELD = new ParseField("real_word_error_likelihood");
+    protected static final ParseField SEPARATOR_FIELD = new ParseField("separator");
+    protected static final ParseField CONFIDENCE_FIELD = new ParseField("confidence");
+    protected static final ParseField GENERATORS_FIELD = new ParseField("shard_size");
+    protected static final ParseField GRAMSIZE_FIELD = new ParseField("gram_size");
+    protected static final ParseField SMOOTHING_MODEL_FIELD = new ParseField("smoothing");
+    protected static final ParseField FORCE_UNIGRAM_FIELD = new ParseField("force_unigrams");
+    protected static final ParseField TOKEN_LIMIT_FIELD = new ParseField("token_limit");
+    protected static final ParseField HIGHLIGHT_FIELD = new ParseField("highlight");
+    protected static final ParseField PRE_TAG_FIELD = new ParseField("pre_tag");
+    protected static final ParseField POST_TAG_FIELD = new ParseField("post_tag");
+    protected static final ParseField COLLATE_FIELD = new ParseField("collate");
+    protected static final ParseField COLLATE_QUERY_FIELD = new ParseField("query");
+    protected static final ParseField COLLATE_QUERY_PARAMS = new ParseField("params");
+    protected static final ParseField COLLATE_QUERY_PRUNE = new ParseField("prune");
+
+    private float maxErrors = PhraseSuggestionContext.DEFAULT_MAX_ERRORS;
+    private String separator = PhraseSuggestionContext.DEFAULT_SEPARATOR;
+    private float realWordErrorLikelihood = PhraseSuggestionContext.DEFAULT_RWE_ERRORLIKELIHOOD;
+    private float confidence = PhraseSuggestionContext.DEFAULT_CONFIDENCE;
+    // gramSize needs to be optional although there is a default, if unset parser try to detect and use shingle size
     private Integer gramSize;
-    private SmoothingModel model;
-    private Boolean forceUnigrams;
-    private Integer tokenLimit;
+    private boolean forceUnigrams = PhraseSuggestionContext.DEFAULT_REQUIRE_UNIGRAM;
+    private int tokenLimit = NoisyChannelSpellChecker.DEFAULT_TOKEN_LIMIT;
     private String preTag;
     private String postTag;
     private Template collateQuery;
     private Map<String, Object> collateParams;
-    private Boolean collatePrune;
+    private boolean collatePrune = PhraseSuggestionContext.DEFAULT_COLLATE_PRUNE;
+    private SmoothingModel model;
+    private final Map<String, List<CandidateGenerator>> generators = new HashMap<>();
 
-    public PhraseSuggestionBuilder(String name) {
-        super(name, "phrase");
+    public PhraseSuggestionBuilder(String field) {
+        super(field);
+    }
+
+    /**
+     * internal copy constructor that copies over all class fields except for the field which is
+     * set to the one provided in the first argument
+     */
+    private PhraseSuggestionBuilder(String fieldname, PhraseSuggestionBuilder in) {
+        super(fieldname, in);
+        maxErrors = in.maxErrors;
+        separator = in.separator;
+        realWordErrorLikelihood = in.realWordErrorLikelihood;
+        confidence = in.confidence;
+        gramSize = in.gramSize;
+        forceUnigrams = in.forceUnigrams;
+        tokenLimit = in.tokenLimit;
+        preTag = in.preTag;
+        postTag = in.postTag;
+        collateQuery = in.collateQuery;
+        collateParams = in.collateParams;
+        collatePrune = in.collatePrune;
+        model = in.model;
+        generators.putAll(in.generators);
     }
 
     /**
@@ -81,6 +134,13 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
         return this;
     }
 
+    /**
+     * get the {@link #gramSize(int)} parameter
+     */
+    public Integer gramSize() {
+        return this.gramSize;
+    }
+
     /**
      * Sets the maximum percentage of the terms that at most considered to be
      * misspellings in order to form a correction. This method accepts a float
@@ -90,30 +150,58 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
      * The default is set to <tt>1.0</tt> which corresponds to that only
      * corrections with at most 1 misspelled term are returned.
      */
-    public PhraseSuggestionBuilder maxErrors(Float maxErrors) {
+    public PhraseSuggestionBuilder maxErrors(float maxErrors) {
+        if (maxErrors <= 0.0) {
+            throw new IllegalArgumentException("max_error must be > 0.0");
+        }
         this.maxErrors = maxErrors;
         return this;
     }
 
+    /**
+     * get the maxErrors setting
+     */
+    public Float maxErrors() {
+        return this.maxErrors;
+    }
+
     /**
      * Sets the separator that is used to separate terms in the bigram field. If
      * not set the whitespace character is used as a separator.
      */
     public PhraseSuggestionBuilder separator(String separator) {
+        Objects.requireNonNull(separator, "separator cannot be set to null");
         this.separator = separator;
         return this;
     }
 
+    /**
+     * get the separator that is used to separate terms in the bigram field.
+     */
+    public String separator() {
+        return this.separator;
+    }
+
     /**
      * Sets the likelihood of a term being a misspelled even if the term exists
      * in the dictionary. The default it <tt>0.95</tt> corresponding to 5% or
      * the real words are misspelled.
      */
-    public PhraseSuggestionBuilder realWordErrorLikelihood(Float realWordErrorLikelihood) {
+    public PhraseSuggestionBuilder realWordErrorLikelihood(float realWordErrorLikelihood) {
+        if (realWordErrorLikelihood <= 0.0) {
+            throw new IllegalArgumentException("real_word_error_likelihood must be > 0.0");
+        }
         this.realWordErrorLikelihood = realWordErrorLikelihood;
         return this;
     }
 
+    /**
+     * get the {@link #realWordErrorLikelihood(float)} parameter
+     */
+    public Float realWordErrorLikelihood() {
+        return this.realWordErrorLikelihood;
+    }
+
     /**
      * Sets the confidence level for this suggester. The confidence level
      * defines a factor applied to the input phrases score which is used as a
@@ -123,11 +211,21 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
      * higher than the input phrase. If set to <tt>0.0</tt> the top N candidates
      * are returned. The default is <tt>1.0</tt>
      */
-    public PhraseSuggestionBuilder confidence(Float confidence) {
+    public PhraseSuggestionBuilder confidence(float confidence) {
+        if (confidence < 0.0) {
+            throw new IllegalArgumentException("confidence must be >= 0.0");
+        }
         this.confidence = confidence;
         return this;
     }
 
+    /**
+     * get the {@link #confidence()} parameter
+     */
+    public Float confidence() {
+        return this.confidence;
+    }
+
     /**
      * Adds a {@link CandidateGenerator} to this suggester. The
      * {@link CandidateGenerator} is used to draw candidates for each individual
@@ -160,20 +258,44 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
         return this;
     }
 
+    /**
+     * get the setting for {@link #forceUnigrams()}
+     */
+    public Boolean forceUnigrams() {
+        return this.forceUnigrams;
+    }
+
     /**
      * Sets an explicit smoothing model used for this suggester. The default is
-     * {@link PhraseSuggestionBuilder.StupidBackoff}.
+     * {@link StupidBackoff}.
      */
     public PhraseSuggestionBuilder smoothingModel(SmoothingModel model) {
         this.model = model;
         return this;
     }
 
+    /**
+     * Gets the {@link SmoothingModel}
+     */
+    public SmoothingModel smoothingModel() {
+        return this.model;
+    }
+
     public PhraseSuggestionBuilder tokenLimit(int tokenLimit) {
+        if (tokenLimit <= 0) {
+            throw new IllegalArgumentException("token_limit must be >= 1");
+        }
         this.tokenLimit = tokenLimit;
         return this;
     }
 
+    /**
+     * get the {@link #tokenLimit(int)} parameter
+     */
+    public Integer tokenLimit() {
+        return this.tokenLimit;
+    }
+
     /**
      * Setup highlighting for suggestions.  If this is called a highlight field
      * is returned with suggestions wrapping changed tokens with preTag and postTag.
@@ -187,6 +309,20 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
         return this;
     }
 
+    /**
+     * get the pre-tag for the highlighter set with {@link #highlight(String, String)}
+     */
+    public String preTag() {
+        return this.preTag;
+    }
+
+    /**
+     * get the post-tag for the highlighter set with {@link #highlight(String, String)}
+     */
+    public String postTag() {
+        return this.postTag;
+    }
+
     /**
      * Sets a query used for filtering out suggested phrases (collation).
      */
@@ -204,13 +340,29 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
     }
 
     /**
-     * Sets additional params for collate script
+     * gets the query used for filtering out suggested phrases (collation).
+     */
+    public Template collateQuery() {
+        return this.collateQuery;
+    }
+
+    /**
+     * Adds additional parameters for collate scripts. Previously added parameters on the
+     * same builder will be overwritten.
      */
     public PhraseSuggestionBuilder collateParams(Map<String, Object> collateParams) {
-        this.collateParams = collateParams;
+        Objects.requireNonNull(collateParams, "collate parameters cannot be null.");
+        this.collateParams = new HashMap<>(collateParams);
         return this;
     }
 
+    /**
+     * gets additional params for collate script
+     */
+    public Map<String, Object> collateParams() {
+        return this.collateParams;
+    }
+
     /**
      * Sets whether to prune suggestions after collation
      */
@@ -219,29 +371,24 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
         return this;
     }
 
+    /**
+     * Gets whether to prune suggestions after collation
+     */
+    public Boolean collatePrune() {
+        return this.collatePrune;
+    }
+
     @Override
     public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
-        if (realWordErrorLikelihood != null) {
-            builder.field("real_word_error_likelihood", realWordErrorLikelihood);
-        }
-        if (confidence != null) {
-            builder.field("confidence", confidence);
-        }
-        if (separator != null) {
-            builder.field("separator", separator);
-        }
-        if (maxErrors != null) {
-            builder.field("max_errors", maxErrors);
-        }
+        builder.field(RWE_LIKELIHOOD_FIELD.getPreferredName(), realWordErrorLikelihood);
+        builder.field(CONFIDENCE_FIELD.getPreferredName(), confidence);
+        builder.field(SEPARATOR_FIELD.getPreferredName(), separator);
+        builder.field(MAXERRORS_FIELD.getPreferredName(), maxErrors);
         if (gramSize != null) {
-            builder.field("gram_size", gramSize);
-        }
-        if (forceUnigrams != null) {
-            builder.field("force_unigrams", forceUnigrams);
-        }
-        if (tokenLimit != null) {
-            builder.field("token_limit", tokenLimit);
+            builder.field(GRAMSIZE_FIELD.getPreferredName(), gramSize);
         }
+        builder.field(FORCE_UNIGRAM_FIELD.getPreferredName(), forceUnigrams);
+        builder.field(TOKEN_LIMIT_FIELD.getPreferredName(), tokenLimit);
         if (!generators.isEmpty()) {
             Set<Entry<String, List<CandidateGenerator>>> entrySet = generators.entrySet();
             for (Entry<String, List<CandidateGenerator>> entry : entrySet) {
@@ -253,412 +400,331 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
             }
         }
         if (model != null) {
-            builder.startObject("smoothing");
+            builder.startObject(SMOOTHING_MODEL_FIELD.getPreferredName());
             model.toXContent(builder, params);
             builder.endObject();
         }
         if (preTag != null) {
-            builder.startObject("highlight");
-            builder.field("pre_tag", preTag);
-            builder.field("post_tag", postTag);
+            builder.startObject(HIGHLIGHT_FIELD.getPreferredName());
+            builder.field(PRE_TAG_FIELD.getPreferredName(), preTag);
+            builder.field(POST_TAG_FIELD.getPreferredName(), postTag);
             builder.endObject();
         }
         if (collateQuery != null) {
-            builder.startObject("collate");
-            builder.field("query", collateQuery);
+            builder.startObject(COLLATE_FIELD.getPreferredName());
+            builder.field(COLLATE_QUERY_FIELD.getPreferredName(), collateQuery);
             if (collateParams != null) {
-                builder.field("params", collateParams);
-            }
-            if (collatePrune != null) {
-                builder.field("prune", collatePrune.booleanValue());
+                builder.field(COLLATE_QUERY_PARAMS.getPreferredName(), collateParams);
             }
+            builder.field(COLLATE_QUERY_PRUNE.getPreferredName(), collatePrune);
             builder.endObject();
         }
         return builder;
     }
 
-    /**
-     * Creates a new {@link DirectCandidateGeneratorBuilder}
-     *
-     * @param field
-     *            the field this candidate generator operates on.
-     */
-    public static DirectCandidateGeneratorBuilder candidateGenerator(String field) {
-        return new DirectCandidateGeneratorBuilder(field);
-    }
-
-    /**
-     * A "stupid-backoff" smoothing model similar to <a
-     * href="http://en.wikipedia.org/wiki/Katz's_back-off_model"> Katz's
-     * Backoff</a>. This model is used as the default if no model is configured.
-     * <p>
-     * See <a
-     * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
-     * Smoothing</a> for details.
-     * </p>
-     */
-    public static final class StupidBackoff extends SmoothingModel {
-        /**
-         * Default discount parameter for {@link StupidBackoff} smoothing
-         */
-        public static final double DEFAULT_BACKOFF_DISCOUNT = 0.4;
-        private double discount = DEFAULT_BACKOFF_DISCOUNT;
-        static final StupidBackoff PROTOTYPE = new StupidBackoff(DEFAULT_BACKOFF_DISCOUNT);
-        private static final String NAME = "stupid_backoff";
-        private static final ParseField DISCOUNT_FIELD = new ParseField("discount");
-
-        /**
-         * Creates a Stupid-Backoff smoothing model.
-         *
-         * @param discount
-         *            the discount given to lower order ngrams if the higher order ngram doesn't exits
-         */
-        public StupidBackoff(double discount) {
-            this.discount = discount;
-        }
-
-        /**
-         * @return the discount parameter of the model
-         */
-        public double getDiscount() {
-            return this.discount;
-        }
-
-        @Override
-        protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field(DISCOUNT_FIELD.getPreferredName(), discount);
-            return builder;
-        }
-
-        @Override
-        public String getWriteableName() {
-            return NAME;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeDouble(discount);
-        }
-
-        @Override
-        public StupidBackoff readFrom(StreamInput in) throws IOException {
-            return new StupidBackoff(in.readDouble());
-        }
-
-        @Override
-        protected boolean doEquals(SmoothingModel other) {
-            StupidBackoff otherModel = (StupidBackoff) other;
-            return Objects.equals(discount, otherModel.discount);
-        }
-
-        @Override
-        protected final int doHashCode() {
-            return Objects.hash(discount);
-        }
-
-        @Override
-        public SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException {
-            XContentParser parser = parseContext.parser();
-            XContentParser.Token token;
-            String fieldName = null;
-            double discount = DEFAULT_BACKOFF_DISCOUNT;
-            while ((token = parser.nextToken()) != Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    fieldName = parser.currentName();
+    @Override
+    protected PhraseSuggestionBuilder innerFromXContent(QueryParseContext parseContext) throws IOException {
+        XContentParser parser = parseContext.parser();
+        PhraseSuggestionBuilder tmpSuggestion = new PhraseSuggestionBuilder("_na_");
+        ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher();
+        XContentParser.Token token;
+        String currentFieldName = null;
+        String fieldname = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.ANALYZER_FIELD)) {
+                    tmpSuggestion.analyzer(parser.text());
+                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.FIELDNAME_FIELD)) {
+                    fieldname = parser.text();
+                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SIZE_FIELD)) {
+                    tmpSuggestion.size(parser.intValue());
+                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SHARDSIZE_FIELD)) {
+                    tmpSuggestion.shardSize(parser.intValue());
+                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.RWE_LIKELIHOOD_FIELD)) {
+                    tmpSuggestion.realWordErrorLikelihood(parser.floatValue());
+                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.CONFIDENCE_FIELD)) {
+                    tmpSuggestion.confidence(parser.floatValue());
+                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.SEPARATOR_FIELD)) {
+                    tmpSuggestion.separator(parser.text());
+                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.MAXERRORS_FIELD)) {
+                    tmpSuggestion.maxErrors(parser.floatValue());
+                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.GRAMSIZE_FIELD)) {
+                    tmpSuggestion.gramSize(parser.intValue());
+                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.FORCE_UNIGRAM_FIELD)) {
+                    tmpSuggestion.forceUnigrams(parser.booleanValue());
+                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.TOKEN_LIMIT_FIELD)) {
+                    tmpSuggestion.tokenLimit(parser.intValue());
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(),
+                            "suggester[phrase] doesn't support field [" + currentFieldName + "]");
                 }
-                if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, DISCOUNT_FIELD)) {
-                    discount = parser.doubleValue();
-                }
-            }
-            return new StupidBackoff(discount);
-        }
-
-        @Override
-        public WordScorerFactory buildWordScorerFactory() {
-            return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
-                    -> new StupidBackoffScorer(reader, terms, field, realWordLikelyhood, separator, discount);
-        }
-    }
-
-    /**
-     * An <a href="http://en.wikipedia.org/wiki/Additive_smoothing">additive
-     * smoothing</a> model.
-     * <p>
-     * See <a
-     * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
-     * Smoothing</a> for details.
-     * </p>
-     */
-    public static final class Laplace extends SmoothingModel {
-        private double alpha = DEFAULT_LAPLACE_ALPHA;
-        private static final String NAME = "laplace";
-        private static final ParseField ALPHA_FIELD = new ParseField("alpha");
-        /**
-         * Default alpha parameter for laplace smoothing
-         */
-        public static final double DEFAULT_LAPLACE_ALPHA = 0.5;
-        static final Laplace PROTOTYPE = new Laplace(DEFAULT_LAPLACE_ALPHA);
-
-        /**
-         * Creates a Laplace smoothing model.
-         *
-         */
-        public Laplace(double alpha) {
-            this.alpha = alpha;
-        }
-
-        /**
-         * @return the laplace model alpha parameter
-         */
-        public double getAlpha() {
-            return this.alpha;
-        }
-
-        @Override
-        protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field(ALPHA_FIELD.getPreferredName(), alpha);
-            return builder;
-        }
-
-        @Override
-        public String getWriteableName() {
-            return NAME;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeDouble(alpha);
-        }
-
-        @Override
-        public SmoothingModel readFrom(StreamInput in) throws IOException {
-            return new Laplace(in.readDouble());
-        }
-
-        @Override
-        protected boolean doEquals(SmoothingModel other) {
-            Laplace otherModel = (Laplace) other;
-            return Objects.equals(alpha, otherModel.alpha);
-        }
-
-        @Override
-        protected final int doHashCode() {
-            return Objects.hash(alpha);
-        }
-
-        @Override
-        public SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException {
-            XContentParser parser = parseContext.parser();
-            XContentParser.Token token;
-            String fieldName = null;
-            double alpha = DEFAULT_LAPLACE_ALPHA;
-            while ((token = parser.nextToken()) != Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    fieldName = parser.currentName();
-                }
-                if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, ALPHA_FIELD)) {
-                    alpha = parser.doubleValue();
-                }
-            }
-            return new Laplace(alpha);
-        }
-
-        @Override
-        public WordScorerFactory buildWordScorerFactory() {
-            return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
-                    -> new LaplaceScorer(reader, terms,  field, realWordLikelyhood, separator, alpha);
-        }
-    }
-
-
-    public static abstract class SmoothingModel implements NamedWriteable<SmoothingModel>, ToXContent {
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(getWriteableName());
-            innerToXContent(builder,params);
-            builder.endObject();
-            return builder;
-        }
-
-        @Override
-        public final boolean equals(Object obj) {
-            if (this == obj) {
-                return true;
-            }
-            if (obj == null || getClass() != obj.getClass()) {
-                return false;
-            }
-            SmoothingModel other = (SmoothingModel) obj;
-            return doEquals(other);
-        }
-
-        @Override
-        public final int hashCode() {
-            /*
-             * Override hashCode here and forward to an abstract method to force extensions of this class to override hashCode in the same
-             * way that we force them to override equals. This also prevents false positives in CheckStyle's EqualsHashCode check.
-             */
-            return doHashCode();
-        }
-
-        public abstract SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException;
-
-        public abstract WordScorerFactory buildWordScorerFactory();
-
-        /**
-         * subtype specific implementation of "equals".
-         */
-        protected abstract boolean doEquals(SmoothingModel other);
-
-        protected abstract int doHashCode();
-
-        protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException;
-    }
-
-    /**
-     * Linear interpolation smoothing model.
-     * <p>
-     * See <a
-     * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
-     * Smoothing</a> for details.
-     * </p>
-     */
-    public static final class LinearInterpolation extends SmoothingModel {
-        private static final String NAME = "linear";
-        static final LinearInterpolation PROTOTYPE = new LinearInterpolation(0.8, 0.1, 0.1);
-        private final double trigramLambda;
-        private final double bigramLambda;
-        private final double unigramLambda;
-        private static final ParseField TRIGRAM_FIELD = new ParseField("trigram_lambda");
-        private static final ParseField BIGRAM_FIELD = new ParseField("bigram_lambda");
-        private static final ParseField UNIGRAM_FIELD = new ParseField("unigram_lambda");
-
-        /**
-         * Creates a linear interpolation smoothing model.
-         *
-         * Note: the lambdas must sum up to one.
-         *
-         * @param trigramLambda
-         *            the trigram lambda
-         * @param bigramLambda
-         *            the bigram lambda
-         * @param unigramLambda
-         *            the unigram lambda
-         */
-        public LinearInterpolation(double trigramLambda, double bigramLambda, double unigramLambda) {
-            double sum = trigramLambda + bigramLambda + unigramLambda;
-            if (Math.abs(sum - 1.0) > 0.001) {
-                throw new IllegalArgumentException("linear smoothing lambdas must sum to 1");
-            }
-            this.trigramLambda = trigramLambda;
-            this.bigramLambda = bigramLambda;
-            this.unigramLambda = unigramLambda;
-        }
-
-        public double getTrigramLambda() {
-            return this.trigramLambda;
-        }
-
-        public double getBigramLambda() {
-            return this.bigramLambda;
-        }
-
-        public double getUnigramLambda() {
-            return this.unigramLambda;
-        }
-
-        @Override
-        protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field(TRIGRAM_FIELD.getPreferredName(), trigramLambda);
-            builder.field(BIGRAM_FIELD.getPreferredName(), bigramLambda);
-            builder.field(UNIGRAM_FIELD.getPreferredName(), unigramLambda);
-            return builder;
-        }
-
-        @Override
-        public String getWriteableName() {
-            return NAME;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeDouble(trigramLambda);
-            out.writeDouble(bigramLambda);
-            out.writeDouble(unigramLambda);
-        }
-
-        @Override
-        public LinearInterpolation readFrom(StreamInput in) throws IOException {
-            return new LinearInterpolation(in.readDouble(), in.readDouble(), in.readDouble());
-        }
-
-        @Override
-        protected boolean doEquals(SmoothingModel other) {
-            final LinearInterpolation otherModel = (LinearInterpolation) other;
-            return Objects.equals(trigramLambda, otherModel.trigramLambda) &&
-                    Objects.equals(bigramLambda, otherModel.bigramLambda) &&
-                    Objects.equals(unigramLambda, otherModel.unigramLambda);
-        }
-
-        @Override
-        protected final int doHashCode() {
-            return Objects.hash(trigramLambda, bigramLambda, unigramLambda);
-        }
-
-        @Override
-        public LinearInterpolation fromXContent(QueryParseContext parseContext) throws IOException {
-            XContentParser parser = parseContext.parser();
-            XContentParser.Token token;
-            String fieldName = null;
-            double trigramLambda = 0.0;
-            double bigramLambda = 0.0;
-            double unigramLambda = 0.0;
-            ParseFieldMatcher matcher = parseContext.parseFieldMatcher();
-            while ((token = parser.nextToken()) != Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    fieldName = parser.currentName();
-                } else if (token.isValue()) {
-                    if (matcher.match(fieldName, TRIGRAM_FIELD)) {
-                        trigramLambda = parser.doubleValue();
-                        if (trigramLambda < 0) {
-                            throw new IllegalArgumentException("trigram_lambda must be positive");
-                        }
-                    } else if (matcher.match(fieldName, BIGRAM_FIELD)) {
-                        bigramLambda = parser.doubleValue();
-                        if (bigramLambda < 0) {
-                            throw new IllegalArgumentException("bigram_lambda must be positive");
-                        }
-                    } else if (matcher.match(fieldName, UNIGRAM_FIELD)) {
-                        unigramLambda = parser.doubleValue();
-                        if (unigramLambda < 0) {
-                            throw new IllegalArgumentException("unigram_lambda must be positive");
-                        }
-                    } else {
-                        throw new IllegalArgumentException(
-                                "suggester[phrase][smoothing][linear] doesn't support field [" + fieldName + "]");
+            } else if (token == Token.START_ARRAY) {
+                if (parseFieldMatcher.match(currentFieldName, DirectCandidateGeneratorBuilder.DIRECT_GENERATOR_FIELD)) {
+                    // for now we only have a single type of generators
+                    while ((token = parser.nextToken()) == Token.START_OBJECT) {
+                        tmpSuggestion.addCandidateGenerator(DirectCandidateGeneratorBuilder.PROTOTYPE.fromXContent(parseContext));
                     }
                 } else {
-                    throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] unknown token [" + token + "] after [" + fieldName + "]");
+                    throw new ParsingException(parser.getTokenLocation(),
+                            "suggester[phrase]  doesn't support array field [" + currentFieldName + "]");
                 }
+            } else if (token == Token.START_OBJECT) {
+                if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.SMOOTHING_MODEL_FIELD)) {
+                    ensureNoSmoothing(tmpSuggestion);
+                    tmpSuggestion.smoothingModel(SmoothingModel.fromXContent(parseContext));
+                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.HIGHLIGHT_FIELD)) {
+                    String preTag = null;
+                    String postTag = null;
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        if (token == XContentParser.Token.FIELD_NAME) {
+                            currentFieldName = parser.currentName();
+                        } else if (token.isValue()) {
+                            if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.PRE_TAG_FIELD)) {
+                                preTag = parser.text();
+                            } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.POST_TAG_FIELD)) {
+                                postTag = parser.text();
+                            } else {
+                                throw new ParsingException(parser.getTokenLocation(),
+                                    "suggester[phrase][highlight] doesn't support field [" + currentFieldName + "]");
+                            }
+                        }
+                    }
+                    tmpSuggestion.highlight(preTag, postTag);
+                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.COLLATE_FIELD)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        if (token == XContentParser.Token.FIELD_NAME) {
+                            currentFieldName = parser.currentName();
+                        } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.COLLATE_QUERY_FIELD)) {
+                            if (tmpSuggestion.collateQuery() != null) {
+                                throw new ParsingException(parser.getTokenLocation(),
+                                        "suggester[phrase][collate] query already set, doesn't support additional ["
+                                        + currentFieldName + "]");
+                            }
+                            Template template = Template.parse(parser, parseFieldMatcher);
+                            tmpSuggestion.collateQuery(template);
+                        } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.COLLATE_QUERY_PARAMS)) {
+                            tmpSuggestion.collateParams(parser.map());
+                        } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.COLLATE_QUERY_PRUNE)) {
+                            if (parser.isBooleanValue()) {
+                                tmpSuggestion.collatePrune(parser.booleanValue());
+                            } else {
+                                throw new ParsingException(parser.getTokenLocation(),
+                                        "suggester[phrase][collate] prune must be either 'true' or 'false'");
+                            }
+                        } else {
+                            throw new ParsingException(parser.getTokenLocation(),
+                                    "suggester[phrase][collate] doesn't support field [" + currentFieldName + "]");
+                        }
+                    }
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(),
+                            "suggester[phrase]  doesn't support array field [" + currentFieldName + "]");
+                }
+            } else {
+                throw new ParsingException(parser.getTokenLocation(),
+                        "suggester[phrase] doesn't support field [" + currentFieldName + "]");
             }
-            return new LinearInterpolation(trigramLambda, bigramLambda, unigramLambda);
         }
 
-        @Override
-        public WordScorerFactory buildWordScorerFactory() {
-            return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) ->
-                        new LinearInterpoatingScorer(reader, terms, field, realWordLikelyhood, separator, trigramLambda, bigramLambda,
-                            unigramLambda);
+        // now we should have field name, check and copy fields over to the suggestion builder we return
+        if (fieldname == null) {
+            throw new ElasticsearchParseException(
+                "the required field option [" + SuggestUtils.Fields.FIELD.getPreferredName() + "] is missing");
         }
+        return new PhraseSuggestionBuilder(fieldname, tmpSuggestion);
+    }
+
+
+    @Override
+    public SuggestionContext build(QueryShardContext context) throws IOException {
+        PhraseSuggestionContext suggestionContext = new PhraseSuggestionContext(context);
+        MapperService mapperService = context.getMapperService();
+        // copy over common settings to each suggestion builder
+        populateCommonFields(mapperService, suggestionContext);
+
+        suggestionContext.setSeparator(BytesRefs.toBytesRef(this.separator));
+        suggestionContext.setRealWordErrorLikelihood(this.realWordErrorLikelihood);
+        suggestionContext.setConfidence(this.confidence);
+        suggestionContext.setMaxErrors(this.maxErrors);
+        suggestionContext.setSeparator(BytesRefs.toBytesRef(this.separator));
+        suggestionContext.setRequireUnigram(this.forceUnigrams);
+        suggestionContext.setTokenLimit(this.tokenLimit);
+        suggestionContext.setPreTag(BytesRefs.toBytesRef(this.preTag));
+        suggestionContext.setPostTag(BytesRefs.toBytesRef(this.postTag));
+
+        if (this.gramSize != null) {
+            suggestionContext.setGramSize(this.gramSize);
+        }
+
+        for (List<CandidateGenerator> candidateGenerators : this.generators.values()) {
+            for (CandidateGenerator candidateGenerator : candidateGenerators) {
+                suggestionContext.addGenerator(candidateGenerator.build(mapperService));
+            }
+        }
+
+        if (this.model != null) {
+            suggestionContext.setModel(this.model.buildWordScorerFactory());
+        }
+
+        if (this.collateQuery != null) {
+            CompiledScript compiledScript = context.getScriptService().compile(this.collateQuery, ScriptContext.Standard.SEARCH,
+                    Collections.emptyMap());
+            suggestionContext.setCollateQueryScript(compiledScript);
+            if (this.collateParams != null) {
+                suggestionContext.setCollateScriptParams(this.collateParams);
+            }
+            suggestionContext.setCollatePrune(this.collatePrune);
+        }
+
+        if (suggestionContext.model() == null) {
+            suggestionContext.setModel(StupidBackoffScorer.FACTORY);
+        }
+
+        if (this.gramSize == null || suggestionContext.generators().isEmpty()) {
+            final ShingleTokenFilterFactory.Factory shingleFilterFactory = SuggestUtils
+                    .getShingleFilterFactory(suggestionContext.getAnalyzer());
+            if (this.gramSize == null) {
+                // try to detect the shingle size
+                if (shingleFilterFactory != null) {
+                    suggestionContext.setGramSize(shingleFilterFactory.getMaxShingleSize());
+                    if (suggestionContext.getAnalyzer() == null && shingleFilterFactory.getMinShingleSize() > 1
+                            && !shingleFilterFactory.getOutputUnigrams()) {
+                        throw new IllegalArgumentException("The default analyzer for field: [" + suggestionContext.getField()
+                                + "] doesn't emit unigrams. If this is intentional try to set the analyzer explicitly");
+                    }
+                }
+            }
+            if (suggestionContext.generators().isEmpty()) {
+                if (shingleFilterFactory != null && shingleFilterFactory.getMinShingleSize() > 1
+                        && !shingleFilterFactory.getOutputUnigrams() && suggestionContext.getRequireUnigram()) {
+                    throw new IllegalArgumentException("The default candidate generator for phrase suggest can't operate on field: ["
+                            + suggestionContext.getField() + "] since it doesn't emit unigrams. "
+                            + "If this is intentional try to set the candidate generator field explicitly");
+                }
+                // use a default generator on the same field
+                DirectCandidateGenerator generator = new DirectCandidateGenerator();
+                generator.setField(suggestionContext.getField());
+                suggestionContext.addGenerator(generator);
+            }
+        }
+        return suggestionContext;
+    }
+
+    private static void ensureNoSmoothing(PhraseSuggestionBuilder suggestion) {
+        if (suggestion.smoothingModel() != null) {
+            throw new IllegalArgumentException("only one smoothing model supported");
+        }
+    }
+
+    @Override
+    public String getWriteableName() {
+        return SUGGESTION_NAME;
+    }
+
+    @Override
+    public void doWriteTo(StreamOutput out) throws IOException {
+        out.writeFloat(maxErrors);
+        out.writeFloat(realWordErrorLikelihood);
+        out.writeFloat(confidence);
+        out.writeOptionalVInt(gramSize);
+        boolean hasModel = model != null;
+        out.writeBoolean(hasModel);
+        if (hasModel) {
+            out.writePhraseSuggestionSmoothingModel(model);
+        }
+        out.writeBoolean(forceUnigrams);
+        out.writeVInt(tokenLimit);
+        out.writeOptionalString(preTag);
+        out.writeOptionalString(postTag);
+        out.writeString(separator);
+        if (collateQuery != null) {
+            out.writeBoolean(true);
+            collateQuery.writeTo(out);
+        } else {
+            out.writeBoolean(false);
+        }
+        out.writeMap(collateParams);
+        out.writeOptionalBoolean(collatePrune);
+        out.writeVInt(this.generators.size());
+        for (Entry<String, List<CandidateGenerator>> entry : this.generators.entrySet()) {
+            out.writeString(entry.getKey());
+            List<CandidateGenerator> generatorsList = entry.getValue();
+            out.writeVInt(generatorsList.size());
+            for (CandidateGenerator generator : generatorsList) {
+                generator.writeTo(out);
+            }
+        }
+    }
+
+    @Override
+    public PhraseSuggestionBuilder doReadFrom(StreamInput in, String field) throws IOException {
+        PhraseSuggestionBuilder builder = new PhraseSuggestionBuilder(field);
+        builder.maxErrors = in.readFloat();
+        builder.realWordErrorLikelihood = in.readFloat();
+        builder.confidence = in.readFloat();
+        builder.gramSize = in.readOptionalVInt();
+        if (in.readBoolean()) {
+            builder.model = in.readPhraseSuggestionSmoothingModel();
+        }
+        builder.forceUnigrams = in.readBoolean();
+        builder.tokenLimit = in.readVInt();
+        builder.preTag = in.readOptionalString();
+        builder.postTag = in.readOptionalString();
+        builder.separator = in.readString();
+        if (in.readBoolean()) {
+            builder.collateQuery = Template.readTemplate(in);
+        }
+        builder.collateParams = in.readMap();
+        builder.collatePrune = in.readOptionalBoolean();
+        int generatorsEntries = in.readVInt();
+        for (int i = 0; i < generatorsEntries; i++) {
+            String type = in.readString();
+            int numberOfGenerators = in.readVInt();
+            List<CandidateGenerator> generatorsList = new ArrayList<>(numberOfGenerators);
+            for (int g = 0; g < numberOfGenerators; g++) {
+                DirectCandidateGeneratorBuilder generator = DirectCandidateGeneratorBuilder.PROTOTYPE.readFrom(in);
+                generatorsList.add(generator);
+            }
+            builder.generators.put(type, generatorsList);
+        }
+        return builder;
+    }
+
+    @Override
+    protected boolean doEquals(PhraseSuggestionBuilder other) {
+        return Objects.equals(maxErrors, other.maxErrors) &&
+                Objects.equals(separator, other.separator) &&
+                Objects.equals(realWordErrorLikelihood, other.realWordErrorLikelihood) &&
+                Objects.equals(confidence, other.confidence) &&
+                Objects.equals(generators, other.generators) &&
+                Objects.equals(gramSize, other.gramSize) &&
+                Objects.equals(model, other.model) &&
+                Objects.equals(forceUnigrams, other.forceUnigrams) &&
+                Objects.equals(tokenLimit, other.tokenLimit) &&
+                Objects.equals(preTag, other.preTag) &&
+                Objects.equals(postTag, other.postTag) &&
+                Objects.equals(collateQuery, other.collateQuery) &&
+                Objects.equals(collateParams, other.collateParams) &&
+                Objects.equals(collatePrune, other.collatePrune);
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(maxErrors, separator, realWordErrorLikelihood, confidence,
+                generators, gramSize, model, forceUnigrams, tokenLimit, preTag, postTag,
+                collateQuery, collateParams, collatePrune);
     }
 
     /**
      * {@link CandidateGenerator} interface.
      */
-    public interface CandidateGenerator extends ToXContent {
+    public interface CandidateGenerator extends Writeable<CandidateGenerator>, ToXContent {
         String getType();
 
         CandidateGenerator fromXContent(QueryParseContext parseContext) throws IOException;
+
+        PhraseSuggestionContext.DirectCandidateGenerator build(MapperService mapperService) throws IOException;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java
index 736b297fab7..80ac850a38c 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java
@@ -20,9 +20,9 @@ package org.elasticsearch.search.suggest.phrase;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
-import org.elasticsearch.search.suggest.Suggester;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 
 import java.util.ArrayList;
@@ -31,26 +31,31 @@ import java.util.List;
 import java.util.Map;
 
 class PhraseSuggestionContext extends SuggestionContext {
-    private final BytesRef SEPARATOR = new BytesRef(" ");
-    private float maxErrors = 0.5f;
-    private BytesRef separator = SEPARATOR;
-    private float realworldErrorLikelihood = 0.95f;
-    private List<DirectCandidateGenerator> generators = new ArrayList<>();
-    private int gramSize = 1;
-    private float confidence = 1.0f;
+    static final boolean DEFAULT_COLLATE_PRUNE = false;
+    static final boolean DEFAULT_REQUIRE_UNIGRAM = true;
+    static final float DEFAULT_CONFIDENCE = 1.0f;
+    static final int DEFAULT_GRAM_SIZE = 1;
+    static final float DEFAULT_RWE_ERRORLIKELIHOOD = 0.95f;
+    static final float DEFAULT_MAX_ERRORS = 0.5f;
+    static final String DEFAULT_SEPARATOR = " ";
+
+    private float maxErrors = DEFAULT_MAX_ERRORS;
+    private BytesRef separator = new BytesRef(DEFAULT_SEPARATOR);
+    private float realworldErrorLikelihood = DEFAULT_RWE_ERRORLIKELIHOOD;
+    private int gramSize = DEFAULT_GRAM_SIZE;
+    private float confidence = DEFAULT_CONFIDENCE;
     private int tokenLimit = NoisyChannelSpellChecker.DEFAULT_TOKEN_LIMIT;
+    private boolean requireUnigram = DEFAULT_REQUIRE_UNIGRAM;
     private BytesRef preTag;
     private BytesRef postTag;
     private CompiledScript collateQueryScript;
+    private boolean prune = DEFAULT_COLLATE_PRUNE;
+    private List<DirectCandidateGenerator> generators = new ArrayList<>();
     private Map<String, Object> collateScriptParams = new HashMap<>(1);
-
     private WordScorer.WordScorerFactory scorer;
 
-    private boolean requireUnigram = true;
-    private boolean prune = false;
-
-    public PhraseSuggestionContext(Suggester<? extends PhraseSuggestionContext> suggester) {
-        super(suggester);
+    public PhraseSuggestionContext(QueryShardContext shardContext) {
+        super(PhraseSuggester.PROTOTYPE, shardContext);
     }
 
     public float maxErrors() {
@@ -149,8 +154,6 @@ class PhraseSuggestionContext extends SuggestionContext {
         public void postFilter(Analyzer postFilter) {
             this.postFilter = postFilter;
         }
-
-
     }
 
     public void setRequireUnigram(boolean requireUnigram) {
@@ -198,7 +201,7 @@ class PhraseSuggestionContext extends SuggestionContext {
     }
 
     void setCollateScriptParams(Map<String, Object> collateScriptParams) {
-        this.collateScriptParams = collateScriptParams;
+        this.collateScriptParams = new HashMap<>(collateScriptParams);
     }
 
     void setCollatePrune(boolean prune) {
@@ -208,5 +211,4 @@ class PhraseSuggestionContext extends SuggestionContext {
     boolean collatePrune() {
         return prune;
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java
new file mode 100644
index 00000000000..0163c560de4
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.phrase;
+
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
+
+import java.io.IOException;
+
+public abstract class SmoothingModel implements NamedWriteable<SmoothingModel>, ToXContent {
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(getWriteableName());
+        innerToXContent(builder,params);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public final boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        SmoothingModel other = (SmoothingModel) obj;
+        return doEquals(other);
+    }
+
+    @Override
+    public final int hashCode() {
+        /*
+         * Override hashCode here and forward to an abstract method to force
+         * extensions of this class to override hashCode in the same way that we
+         * force them to override equals. This also prevents false positives in
+         * CheckStyle's EqualsHashCode check.
+         */
+        return doHashCode();
+    }
+
+    protected abstract int doHashCode();
+
+    public static SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException {
+        XContentParser parser = parseContext.parser();
+        ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher();
+        XContentParser.Token token;
+        String fieldName = null;
+        SmoothingModel model = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                fieldName = parser.currentName();
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (parseFieldMatcher.match(fieldName, LinearInterpolation.PARSE_FIELD)) {
+                    model = LinearInterpolation.PROTOTYPE.innerFromXContent(parseContext);
+                } else if (parseFieldMatcher.match(fieldName, Laplace.PARSE_FIELD)) {
+                    model = Laplace.PROTOTYPE.innerFromXContent(parseContext);
+                } else if (parseFieldMatcher.match(fieldName, StupidBackoff.PARSE_FIELD)) {
+                    model = StupidBackoff.PROTOTYPE.innerFromXContent(parseContext);
+                } else {
+                    throw new IllegalArgumentException("suggester[phrase] doesn't support object field [" + fieldName + "]");
+                }
+            } else {
+                throw new ParsingException(parser.getTokenLocation(),
+                        "[smoothing] unknown token [" + token + "] after [" + fieldName + "]");
+            }
+        }
+        return model;
+    }
+
+    public abstract SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException;
+
+    public abstract WordScorerFactory buildWordScorerFactory();
+
+    /**
+     * subtype specific implementation of "equals".
+     */
+    protected abstract boolean doEquals(SmoothingModel other);
+
+    protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException;
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java
new file mode 100644
index 00000000000..951f7f917ff
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.phrase;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * A "stupid-backoff" smoothing model similar to <a
+ * href="http://en.wikipedia.org/wiki/Katz's_back-off_model"> Katz's
+ * Backoff</a>. This model is used as the default if no model is configured.
+ * <p>
+ * See <a
+ * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
+ * Smoothing</a> for details.
+ * </p>
+ */
+public final class StupidBackoff extends SmoothingModel {
+    /**
+     * Default discount parameter for {@link StupidBackoff} smoothing
+     */
+    public static final double DEFAULT_BACKOFF_DISCOUNT = 0.4;
+    public static final StupidBackoff PROTOTYPE = new StupidBackoff(DEFAULT_BACKOFF_DISCOUNT);
+    private double discount = DEFAULT_BACKOFF_DISCOUNT;
+    private static final String NAME = "stupid_backoff";
+    private static final ParseField DISCOUNT_FIELD = new ParseField("discount");
+    static final ParseField PARSE_FIELD = new ParseField(NAME);
+
+    /**
+     * Creates a Stupid-Backoff smoothing model.
+     *
+     * @param discount
+     *            the discount given to lower order ngrams if the higher order ngram doesn't exits
+     */
+    public StupidBackoff(double discount) {
+        this.discount = discount;
+    }
+
+    /**
+     * @return the discount parameter of the model
+     */
+    public double getDiscount() {
+        return this.discount;
+    }
+
+    @Override
+    protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(DISCOUNT_FIELD.getPreferredName(), discount);
+        return builder;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeDouble(discount);
+    }
+
+    @Override
+    public StupidBackoff readFrom(StreamInput in) throws IOException {
+        return new StupidBackoff(in.readDouble());
+    }
+
+    @Override
+    protected boolean doEquals(SmoothingModel other) {
+        StupidBackoff otherModel = (StupidBackoff) other;
+        return Objects.equals(discount, otherModel.discount);
+    }
+
+    @Override
+    protected final int doHashCode() {
+        return Objects.hash(discount);
+    }
+
+    @Override
+    public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
+        XContentParser parser = parseContext.parser();
+        XContentParser.Token token;
+        String fieldName = null;
+        double discount = DEFAULT_BACKOFF_DISCOUNT;
+        while ((token = parser.nextToken()) != Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                fieldName = parser.currentName();
+            }
+            if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, DISCOUNT_FIELD)) {
+                discount = parser.doubleValue();
+            }
+        }
+        return new StupidBackoff(discount);
+    }
+
+    @Override
+    public WordScorerFactory buildWordScorerFactory() {
+        return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
+                -> new StupidBackoffScorer(reader, terms, field, realWordLikelyhood, separator, discount);
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/WordScorer.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/WordScorer.java
index 9557715bcb9..69e62c1a175 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/WordScorer.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/WordScorer.java
@@ -77,7 +77,7 @@ public abstract class WordScorer {
        }
        return candidate.stringDistance;
    }
-   
+
    public double score(Candidate[] path, CandidateSet[] candidateSet, int at, int gramSize) throws IOException {
        if (at == 0 || gramSize == 1) {
            return Math.log10(channelScore(path[at], candidateSet[at].originalTerm) * scoreUnigram(path[at]));
@@ -87,21 +87,21 @@ public abstract class WordScorer {
            return Math.log10(channelScore(path[at], candidateSet[at].originalTerm) * scoreTrigram(path[at], path[at - 1], path[at - 2]));
        }
    }
-   
+
    protected double scoreUnigram(Candidate word)  throws IOException {
        return (1.0 + frequency(word.term)) / (vocabluarySize + numTerms);
    }
-   
+
    protected double scoreBigram(Candidate word, Candidate w_1) throws IOException {
        return scoreUnigram(word);
    }
-   
+
    protected double scoreTrigram(Candidate word, Candidate w_1, Candidate w_2) throws IOException {
        return scoreBigram(word, w_1);
    }
 
-   public static interface WordScorerFactory {
-       public WordScorer newScorer(IndexReader reader, Terms terms,
-            String field, double realWordLikelyhood, BytesRef separator) throws IOException;
+   public interface WordScorerFactory {
+       WordScorer newScorer(IndexReader reader, Terms terms,
+                            String field, double realWordLikelyhood, BytesRef separator) throws IOException;
    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java
deleted file mode 100644
index a2fd680c215..00000000000
--- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.suggest.term;
-
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.IndexFieldDataService;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
-import org.elasticsearch.search.suggest.SuggestContextParser;
-import org.elasticsearch.search.suggest.SuggestUtils;
-import org.elasticsearch.search.suggest.SuggestionSearchContext;
-
-import java.io.IOException;
-
-public final class TermSuggestParser implements SuggestContextParser {
-
-    private TermSuggester suggester;
-
-    public TermSuggestParser(TermSuggester suggester) {
-        this.suggester = suggester;
-    }
-
-    @Override
-    public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException {
-        XContentParser.Token token;
-        String fieldName = null;
-        TermSuggestionContext suggestion = new TermSuggestionContext(suggester);
-        DirectSpellcheckerSettings settings = suggestion.getDirectSpellCheckerSettings();
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                fieldName = parser.currentName();
-            } else if (token.isValue()) {
-                parseTokenValue(parser, mapperService, fieldName, suggestion, settings, mapperService.getIndexSettings().getParseFieldMatcher());
-            } else {
-                throw new IllegalArgumentException("suggester[term]  doesn't support field [" + fieldName + "]");
-            }
-        }
-        return suggestion;
-    }
-
-    private void parseTokenValue(XContentParser parser, MapperService mapperService, String fieldName, TermSuggestionContext suggestion,
-            DirectSpellcheckerSettings settings, ParseFieldMatcher parseFieldMatcher) throws IOException {
-        if (!(SuggestUtils.parseSuggestContext(parser, mapperService, fieldName, suggestion, parseFieldMatcher) || SuggestUtils.parseDirectSpellcheckerSettings(
-                parser, fieldName, settings, parseFieldMatcher))) {
-            throw new IllegalArgumentException("suggester[term] doesn't support [" + fieldName + "]");
-
-        }
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java
index 34cd3ad4d56..4bffb2dfe86 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java
@@ -28,9 +28,9 @@ import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.CharsRefBuilder;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.text.Text;
-import org.elasticsearch.search.suggest.SuggestContextParser;
 import org.elasticsearch.search.suggest.SuggestUtils;
 import org.elasticsearch.search.suggest.Suggester;
+import org.elasticsearch.search.suggest.SuggestionBuilder;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 
 import java.io.IOException;
@@ -39,8 +39,11 @@ import java.util.List;
 
 public final class TermSuggester extends Suggester<TermSuggestionContext> {
 
+    public static final TermSuggester PROTOTYPE = new TermSuggester();
+
     @Override
-    public TermSuggestion innerExecute(String name, TermSuggestionContext suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
+    public TermSuggestion innerExecute(String name, TermSuggestionContext suggestion, IndexSearcher searcher, CharsRefBuilder spare)
+            throws IOException {
         DirectSpellChecker directSpellChecker = SuggestUtils.getDirectSpellChecker(suggestion.getDirectSpellCheckerSettings());
         final IndexReader indexReader = searcher.getIndexReader();
         TermSuggestion response = new TermSuggestion(
@@ -63,12 +66,6 @@ public final class TermSuggester extends Suggester<TermSuggestionContext> {
         return response;
     }
 
-    @Override
-    public SuggestContextParser getContextParser() {
-        return new TermSuggestParser(this);
-    }
-
-
     private List<Token> queryTerms(SuggestionContext suggestion, CharsRefBuilder spare) throws IOException {
         final List<Token> result = new ArrayList<>();
         final String field = suggestion.getField();
@@ -76,7 +73,7 @@ public final class TermSuggester extends Suggester<TermSuggestionContext> {
             @Override
             public void nextToken() {
                 Term term = new Term(field, BytesRef.deepCopyOf(fillBytesRef(new BytesRefBuilder())));
-                result.add(new Token(term, offsetAttr.startOffset(), offsetAttr.endOffset())); 
+                result.add(new Token(term, offsetAttr.startOffset(), offsetAttr.endOffset()));
             }
         }, spare);
        return result;
@@ -96,4 +93,9 @@ public final class TermSuggester extends Suggester<TermSuggestionContext> {
 
     }
 
+    @Override
+    public SuggestionBuilder<?> getBuilderPrototype() {
+        return TermSuggestionBuilder.PROTOTYPE;
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestion.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestion.java
index 12f2c93153b..bc4006469ad 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestion.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestion.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentBuilderString;
+import org.elasticsearch.search.suggest.SortBy;
 import org.elasticsearch.search.suggest.Suggest.Suggestion;
 import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
 
@@ -37,6 +38,17 @@ public class TermSuggestion extends Suggestion<TermSuggestion.Entry> {
 
     public static final Comparator<Suggestion.Entry.Option> SCORE = new Score();
     public static final Comparator<Suggestion.Entry.Option> FREQUENCY = new Frequency();
+    public static final int TYPE = 1;
+
+    private SortBy sort;
+
+    public TermSuggestion() {
+    }
+
+    public TermSuggestion(String name, int size, SortBy sort) {
+        super(name, size);
+        this.sort = sort;
+    }
 
     // Same behaviour as comparators in suggest module, but for SuggestedWord
     // Highest score first, then highest freq first, then lowest term first
@@ -79,17 +91,6 @@ public class TermSuggestion extends Suggestion<TermSuggestion.Entry> {
 
     }
 
-    public static final int TYPE = 1;
-    private Sort sort;
-
-    public TermSuggestion() {
-    }
-
-    public TermSuggestion(String name, int size, Sort sort) {
-        super(name, size);
-        this.sort = sort;
-    }
-
     @Override
     public int getType() {
         return TYPE;
@@ -110,13 +111,13 @@ public class TermSuggestion extends Suggestion<TermSuggestion.Entry> {
     @Override
     protected void innerReadFrom(StreamInput in) throws IOException {
         super.innerReadFrom(in);
-        sort = Sort.fromId(in.readByte());
+        sort = SortBy.PROTOTYPE.readFrom(in);
     }
 
     @Override
     public void innerWriteTo(StreamOutput out) throws IOException {
         super.innerWriteTo(out);
-        out.writeByte(sort.id());
+        sort.writeTo(out);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
index 03eb388f003..0cb9d1604a4 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
@@ -16,11 +16,52 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.search.suggest.term;
+
+import org.apache.lucene.search.spell.DirectSpellChecker;
+import org.apache.lucene.search.spell.JaroWinklerDistance;
+import org.apache.lucene.search.spell.LevensteinDistance;
+import org.apache.lucene.search.spell.LuceneLevenshteinDistance;
+import org.apache.lucene.search.spell.NGramDistance;
+import org.apache.lucene.search.spell.StringDistance;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.suggest.SuggestBuilder.SuggestionBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
+import org.elasticsearch.search.suggest.SortBy;
+import org.elasticsearch.search.suggest.SuggestUtils;
+import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 
 import java.io.IOException;
+import java.util.Locale;
+import java.util.Objects;
+
+import static org.elasticsearch.search.suggest.DirectSpellcheckerSettings.DEFAULT_ACCURACY;
+import static org.elasticsearch.search.suggest.DirectSpellcheckerSettings.DEFAULT_MAX_EDITS;
+import static org.elasticsearch.search.suggest.DirectSpellcheckerSettings.DEFAULT_MAX_INSPECTIONS;
+import static org.elasticsearch.search.suggest.DirectSpellcheckerSettings.DEFAULT_MAX_TERM_FREQ;
+import static org.elasticsearch.search.suggest.DirectSpellcheckerSettings.DEFAULT_MIN_DOC_FREQ;
+import static org.elasticsearch.search.suggest.DirectSpellcheckerSettings.DEFAULT_MIN_WORD_LENGTH;
+import static org.elasticsearch.search.suggest.DirectSpellcheckerSettings.DEFAULT_PREFIX_LENGTH;
+import static org.elasticsearch.search.suggest.SuggestUtils.Fields.ACCURACY;
+import static org.elasticsearch.search.suggest.SuggestUtils.Fields.MAX_EDITS;
+import static org.elasticsearch.search.suggest.SuggestUtils.Fields.MAX_INSPECTIONS;
+import static org.elasticsearch.search.suggest.SuggestUtils.Fields.MAX_TERM_FREQ;
+import static org.elasticsearch.search.suggest.SuggestUtils.Fields.MIN_DOC_FREQ;
+import static org.elasticsearch.search.suggest.SuggestUtils.Fields.MIN_WORD_LENGTH;
+import static org.elasticsearch.search.suggest.SuggestUtils.Fields.PREFIX_LENGTH;
+import static org.elasticsearch.search.suggest.SuggestUtils.Fields.SORT;
+import static org.elasticsearch.search.suggest.SuggestUtils.Fields.STRING_DISTANCE;
+import static org.elasticsearch.search.suggest.SuggestUtils.Fields.SUGGEST_MODE;
 
 /**
  * Defines the actual suggest command. Each command uses the global options
@@ -29,23 +70,39 @@ import java.io.IOException;
  */
 public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuilder> {
 
-    private String suggestMode;
-    private Float accuracy;
-    private String sort;
-    private String stringDistance;
-    private Integer maxEdits;
-    private Integer maxInspections;
-    private Float maxTermFreq;
-    private Integer prefixLength;
-    private Integer minWordLength;
-    private Float minDocFreq;
-    
+    public static final TermSuggestionBuilder PROTOTYPE = new TermSuggestionBuilder("_na_");
+    private static final String SUGGESTION_NAME = "term";
+
+    private SuggestMode suggestMode = SuggestMode.MISSING;
+    private float accuracy = DEFAULT_ACCURACY;
+    private SortBy sort = SortBy.SCORE;
+    private StringDistanceImpl stringDistance = StringDistanceImpl.INTERNAL;
+    private int maxEdits = DEFAULT_MAX_EDITS;
+    private int maxInspections = DEFAULT_MAX_INSPECTIONS;
+    private float maxTermFreq = DEFAULT_MAX_TERM_FREQ;
+    private int prefixLength = DEFAULT_PREFIX_LENGTH;
+    private int minWordLength = DEFAULT_MIN_WORD_LENGTH;
+    private float minDocFreq = DEFAULT_MIN_DOC_FREQ;
+
+    public TermSuggestionBuilder(String field) {
+        super(field);
+    }
+
     /**
-     * @param name
-     *            The name of this suggestion. This is a required parameter.
+     * internal copy constructor that copies over all class field except field.
      */
-    public TermSuggestionBuilder(String name) {
-        super(name, "term");
+    private TermSuggestionBuilder(String field, TermSuggestionBuilder in) {
+        super(field, in);
+        suggestMode = in.suggestMode;
+        accuracy = in.accuracy;
+        sort = in.sort;
+        stringDistance = in.stringDistance;
+        maxEdits = in.maxEdits;
+        maxInspections = in.maxInspections;
+        maxTermFreq = in.maxTermFreq;
+        prefixLength = in.prefixLength;
+        minWordLength = in.minWordLength;
+        minDocFreq = in.minDocFreq;
     }
 
     /**
@@ -61,11 +118,19 @@ public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuild
      * tokens in the suggest text.
      * </ol>
      */
-    public TermSuggestionBuilder suggestMode(String suggestMode) {
+    public TermSuggestionBuilder suggestMode(SuggestMode suggestMode) {
+        Objects.requireNonNull(suggestMode, "suggestMode must not be null");
         this.suggestMode = suggestMode;
         return this;
     }
 
+    /**
+     * Get the suggest mode setting.
+     */
+    public SuggestMode suggestMode() {
+        return suggestMode;
+    }
+
     /**
      * s how similar the suggested terms at least need to be compared to the
      * original suggest text tokens. A value between 0 and 1 can be specified.
@@ -74,11 +139,21 @@ public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuild
      * <p>
      * Default is <tt>0.5</tt>
      */
-    public TermSuggestionBuilder setAccuracy(float accuracy) {
+    public TermSuggestionBuilder accuracy(float accuracy) {
+        if (accuracy < 0.0f || accuracy > 1.0f) {
+            throw new IllegalArgumentException("accuracy must be between 0 and 1");
+        }
         this.accuracy = accuracy;
         return this;
     }
 
+    /**
+     * Get the accuracy setting.
+     */
+    public float accuracy() {
+        return accuracy;
+    }
+
     /**
      * Sets how to sort the suggest terms per suggest text token. Two possible
      * values:
@@ -86,19 +161,27 @@ public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuild
      * <li><code>score</code> - Sort should first be based on score, then
      * document frequency and then the term itself.
      * <li><code>frequency</code> - Sort should first be based on document
-     * frequency, then scotr and then the term itself.
+     * frequency, then score and then the term itself.
      * </ol>
      * <p>
      * What the score is depends on the suggester being used.
      */
-    public TermSuggestionBuilder sort(String sort) {
+    public TermSuggestionBuilder sort(SortBy sort) {
+        Objects.requireNonNull(sort, "sort must not be null");
         this.sort = sort;
         return this;
     }
 
+    /**
+     * Get the sort setting.
+     */
+    public SortBy sort() {
+        return sort;
+    }
+
     /**
      * Sets what string distance implementation to use for comparing how similar
-     * suggested terms are. Four possible values can be specified:
+     * suggested terms are. Five possible values can be specified:
      * <ol>
      * <li><code>internal</code> - This is the default and is based on
      * <code>damerau_levenshtein</code>, but highly optimized for comparing
@@ -113,32 +196,60 @@ public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuild
      * n-grams.
      * </ol>
      */
-    public TermSuggestionBuilder stringDistance(String stringDistance) {
+    public TermSuggestionBuilder stringDistance(StringDistanceImpl stringDistance) {
+        Objects.requireNonNull(stringDistance, "stringDistance must not be null");
         this.stringDistance = stringDistance;
         return this;
     }
 
+    /**
+     * Get the string distance implementation setting.
+     */
+    public StringDistanceImpl stringDistance() {
+        return stringDistance;
+    }
+
     /**
      * Sets the maximum edit distance candidate suggestions can have in order to
      * be considered as a suggestion. Can only be a value between 1 and 2. Any
      * other value result in an bad request error being thrown. Defaults to
      * <tt>2</tt>.
      */
-    public TermSuggestionBuilder maxEdits(Integer maxEdits) {
+    public TermSuggestionBuilder maxEdits(int maxEdits) {
+        if (maxEdits < 1 || maxEdits > 2) {
+            throw new IllegalArgumentException("maxEdits must be between 1 and 2");
+        }
         this.maxEdits = maxEdits;
         return this;
     }
 
+    /**
+     * Get the maximum edit distance setting.
+     */
+    public int maxEdits() {
+        return maxEdits;
+    }
+
     /**
      * A factor that is used to multiply with the size in order to inspect more
      * candidate suggestions. Can improve accuracy at the cost of performance.
      * Defaults to <tt>5</tt>.
      */
-    public TermSuggestionBuilder maxInspections(Integer maxInspections) {
+    public TermSuggestionBuilder maxInspections(int maxInspections) {
+        if (maxInspections < 0) {
+            throw new IllegalArgumentException("maxInspections must be positive");
+        }
         this.maxInspections = maxInspections;
         return this;
     }
 
+    /**
+     * Get the factor for inspecting more candidate suggestions setting.
+     */
+    public int maxInspections() {
+        return maxInspections;
+    }
+
     /**
      * Sets a maximum threshold in number of documents a suggest text token can
      * exist in order to be corrected. Can be a relative percentage number (e.g
@@ -151,10 +262,23 @@ public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuild
      * also improves the suggest performance.
      */
     public TermSuggestionBuilder maxTermFreq(float maxTermFreq) {
+        if (maxTermFreq < 0.0f) {
+            throw new IllegalArgumentException("maxTermFreq must be positive");
+        }
+        if (maxTermFreq > 1.0f && maxTermFreq != Math.floor(maxTermFreq)) {
+            throw new IllegalArgumentException("if maxTermFreq is greater than 1, it must not be a fraction");
+        }
         this.maxTermFreq = maxTermFreq;
         return this;
     }
 
+    /**
+     * Get the maximum term frequency threshold setting.
+     */
+    public float maxTermFreq() {
+        return maxTermFreq;
+    }
+
     /**
      * Sets the number of minimal prefix characters that must match in order be
      * a candidate suggestion. Defaults to 1. Increasing this number improves
@@ -162,19 +286,39 @@ public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuild
      * terms.
      */
     public TermSuggestionBuilder prefixLength(int prefixLength) {
+        if (prefixLength < 0) {
+            throw new IllegalArgumentException("prefixLength must be positive");
+        }
         this.prefixLength = prefixLength;
         return this;
     }
 
+    /**
+     * Get the minimum prefix length that must match setting.
+     */
+    public int prefixLength() {
+        return prefixLength;
+    }
+
     /**
      * The minimum length a suggest text term must have in order to be
      * corrected. Defaults to <tt>4</tt>.
      */
     public TermSuggestionBuilder minWordLength(int minWordLength) {
+        if (minWordLength < 1) {
+            throw new IllegalArgumentException("minWordLength must be greater or equal to 1");
+        }
         this.minWordLength = minWordLength;
         return this;
     }
 
+    /**
+     * Get the minimum length of a text term to be corrected setting.
+     */
+    public int minWordLength() {
+        return minWordLength;
+    }
+
     /**
      * Sets a minimal threshold in number of documents a suggested term should
      * appear in. This can be specified as an absolute number or as a relative
@@ -183,42 +327,294 @@ public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuild
      * value higher than 1 is specified then the number cannot be fractional.
      */
     public TermSuggestionBuilder minDocFreq(float minDocFreq) {
+        if (minDocFreq < 0.0f) {
+            throw new IllegalArgumentException("minDocFreq must be positive");
+        }
+        if (minDocFreq > 1.0f && minDocFreq != Math.floor(minDocFreq)) {
+            throw new IllegalArgumentException("if minDocFreq is greater than 1, it must not be a fraction");
+        }
         this.minDocFreq = minDocFreq;
         return this;
     }
 
+    /**
+     * Get the minimal threshold for the frequency of a term appearing in the
+     * document set setting.
+     */
+    public float minDocFreq() {
+        return minDocFreq;
+    }
+
     @Override
     public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
-        if (suggestMode != null) {
-            builder.field("suggest_mode", suggestMode);
-        }
-        if (accuracy != null) {
-            builder.field("accuracy", accuracy);
-        }
-        if (sort != null) {
-            builder.field("sort", sort);
-        }
-        if (stringDistance != null) {
-            builder.field("string_distance", stringDistance);
-        }
-        if (maxEdits != null) {
-            builder.field("max_edits", maxEdits);
-        }
-        if (maxInspections != null) {
-            builder.field("max_inspections", maxInspections);
-        }
-        if (maxTermFreq != null) {
-            builder.field("max_term_freq", maxTermFreq);
-        }
-        if (prefixLength != null) {
-            builder.field("prefix_length", prefixLength);
-        }
-        if (minWordLength != null) {
-            builder.field("min_word_length", minWordLength);
-        }
-        if (minDocFreq != null) {
-            builder.field("min_doc_freq", minDocFreq);
-        }
+        builder.field(SUGGEST_MODE.getPreferredName(), suggestMode);
+        builder.field(ACCURACY.getPreferredName(), accuracy);
+        builder.field(SORT.getPreferredName(), sort);
+        builder.field(STRING_DISTANCE.getPreferredName(), stringDistance);
+        builder.field(MAX_EDITS.getPreferredName(), maxEdits);
+        builder.field(MAX_INSPECTIONS.getPreferredName(), maxInspections);
+        builder.field(MAX_TERM_FREQ.getPreferredName(), maxTermFreq);
+        builder.field(PREFIX_LENGTH.getPreferredName(), prefixLength);
+        builder.field(MIN_WORD_LENGTH.getPreferredName(), minWordLength);
+        builder.field(MIN_DOC_FREQ.getPreferredName(), minDocFreq);
         return builder;
     }
+
+    @Override
+    protected TermSuggestionBuilder innerFromXContent(QueryParseContext parseContext) throws IOException {
+        XContentParser parser = parseContext.parser();
+        TermSuggestionBuilder tmpSuggestion = new TermSuggestionBuilder("_na_");
+        ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher();
+        XContentParser.Token token;
+        String currentFieldName = null;
+        String fieldname = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.ANALYZER_FIELD)) {
+                    tmpSuggestion.analyzer(parser.text());
+                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.FIELDNAME_FIELD)) {
+                    fieldname = parser.text();
+                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SIZE_FIELD)) {
+                    tmpSuggestion.size(parser.intValue());
+                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SHARDSIZE_FIELD)) {
+                    tmpSuggestion.shardSize(parser.intValue());
+                } else if (parseFieldMatcher.match(currentFieldName, SUGGEST_MODE)) {
+                    tmpSuggestion.suggestMode(SuggestMode.resolve(parser.text()));
+                } else if (parseFieldMatcher.match(currentFieldName, ACCURACY)) {
+                    tmpSuggestion.accuracy(parser.floatValue());
+                } else if (parseFieldMatcher.match(currentFieldName, SORT)) {
+                    tmpSuggestion.sort(SortBy.resolve(parser.text()));
+                } else if (parseFieldMatcher.match(currentFieldName, STRING_DISTANCE)) {
+                    tmpSuggestion.stringDistance(StringDistanceImpl.resolve(parser.text()));
+                } else if (parseFieldMatcher.match(currentFieldName, MAX_EDITS)) {
+                    tmpSuggestion.maxEdits(parser.intValue());
+                } else if (parseFieldMatcher.match(currentFieldName, MAX_INSPECTIONS)) {
+                    tmpSuggestion.maxInspections(parser.intValue());
+                } else if (parseFieldMatcher.match(currentFieldName, MAX_TERM_FREQ)) {
+                    tmpSuggestion.maxTermFreq(parser.floatValue());
+                } else if (parseFieldMatcher.match(currentFieldName, PREFIX_LENGTH)) {
+                    tmpSuggestion.prefixLength(parser.intValue());
+                } else if (parseFieldMatcher.match(currentFieldName, MIN_WORD_LENGTH)) {
+                    tmpSuggestion.minWordLength(parser.intValue());
+                } else if (parseFieldMatcher.match(currentFieldName, MIN_DOC_FREQ)) {
+                    tmpSuggestion.minDocFreq(parser.floatValue());
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(),
+                                                  "suggester[term] doesn't support field [" + currentFieldName + "]");
+                }
+            } else {
+                throw new ParsingException(parser.getTokenLocation(), "suggester[term] parsing failed on [" + currentFieldName + "]");
+            }
+        }
+
+        // now we should have field name, check and copy fields over to the suggestion builder we return
+        if (fieldname == null) {
+            throw new ElasticsearchParseException(
+                "the required field option [" + SuggestUtils.Fields.FIELD.getPreferredName() + "] is missing");
+        }
+        return new TermSuggestionBuilder(fieldname, tmpSuggestion);
+    }
+
+    @Override
+    public SuggestionContext build(QueryShardContext context) throws IOException {
+        TermSuggestionContext suggestionContext = new TermSuggestionContext(context);
+        // copy over common settings to each suggestion builder
+        populateCommonFields(context.getMapperService(), suggestionContext);
+        // Transfers the builder settings to the target TermSuggestionContext
+        DirectSpellcheckerSettings settings = suggestionContext.getDirectSpellCheckerSettings();
+        settings.accuracy(accuracy);
+        settings.maxEdits(maxEdits);
+        settings.maxInspections(maxInspections);
+        settings.maxTermFreq(maxTermFreq);
+        settings.minDocFreq(minDocFreq);
+        settings.minWordLength(minWordLength);
+        settings.prefixLength(prefixLength);
+        settings.sort(sort);
+        settings.stringDistance(stringDistance.toLucene());
+        settings.suggestMode(suggestMode.toLucene());
+        return suggestionContext;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return SUGGESTION_NAME;
+    }
+
+    @Override
+    public void doWriteTo(StreamOutput out) throws IOException {
+        suggestMode.writeTo(out);
+        out.writeFloat(accuracy);
+        sort.writeTo(out);
+        stringDistance.writeTo(out);
+        out.writeVInt(maxEdits);
+        out.writeVInt(maxInspections);
+        out.writeFloat(maxTermFreq);
+        out.writeVInt(prefixLength);
+        out.writeVInt(minWordLength);
+        out.writeFloat(minDocFreq);
+    }
+
+    @Override
+    public TermSuggestionBuilder doReadFrom(StreamInput in, String field) throws IOException {
+        TermSuggestionBuilder builder = new TermSuggestionBuilder(field);
+        builder.suggestMode = SuggestMode.PROTOTYPE.readFrom(in);
+        builder.accuracy = in.readFloat();
+        builder.sort = SortBy.PROTOTYPE.readFrom(in);
+        builder.stringDistance = StringDistanceImpl.PROTOTYPE.readFrom(in);
+        builder.maxEdits = in.readVInt();
+        builder.maxInspections = in.readVInt();
+        builder.maxTermFreq = in.readFloat();
+        builder.prefixLength = in.readVInt();
+        builder.minWordLength = in.readVInt();
+        builder.minDocFreq = in.readFloat();
+        return builder;
+    }
+
+    @Override
+    protected boolean doEquals(TermSuggestionBuilder other) {
+        return Objects.equals(suggestMode, other.suggestMode) &&
+               Objects.equals(accuracy, other.accuracy) &&
+               Objects.equals(sort, other.sort) &&
+               Objects.equals(stringDistance, other.stringDistance) &&
+               Objects.equals(maxEdits, other.maxEdits) &&
+               Objects.equals(maxInspections, other.maxInspections) &&
+               Objects.equals(maxTermFreq, other.maxTermFreq) &&
+               Objects.equals(prefixLength, other.prefixLength) &&
+               Objects.equals(minWordLength, other.minWordLength) &&
+               Objects.equals(minDocFreq, other.minDocFreq);
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(suggestMode, accuracy, sort, stringDistance, maxEdits, maxInspections,
+                            maxTermFreq, prefixLength, minWordLength, minDocFreq);
+    }
+
+    /** An enum representing the valid suggest modes. */
+    public enum SuggestMode implements Writeable<SuggestMode> {
+        /** Only suggest terms in the suggest text that aren't in the index. This is the default. */
+        MISSING {
+            @Override
+            public org.apache.lucene.search.spell.SuggestMode toLucene() {
+                return org.apache.lucene.search.spell.SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX;
+            }
+        },
+        /** Only suggest terms that occur in more docs then the original suggest text term. */
+        POPULAR {
+            @Override
+            public org.apache.lucene.search.spell.SuggestMode toLucene() {
+                return org.apache.lucene.search.spell.SuggestMode.SUGGEST_MORE_POPULAR;
+            }
+        },
+        /** Suggest any matching suggest terms based on tokens in the suggest text. */
+        ALWAYS {
+            @Override
+            public org.apache.lucene.search.spell.SuggestMode toLucene() {
+              return org.apache.lucene.search.spell.SuggestMode.SUGGEST_ALWAYS;
+            }
+        };
+
+        protected static SuggestMode PROTOTYPE = MISSING;
+
+        @Override
+        public void writeTo(final StreamOutput out) throws IOException {
+            out.writeVInt(ordinal());
+        }
+
+        @Override
+        public SuggestMode readFrom(final StreamInput in) throws IOException {
+            int ordinal = in.readVInt();
+            if (ordinal < 0 || ordinal >= values().length) {
+                throw new IOException("Unknown SuggestMode ordinal [" + ordinal + "]");
+            }
+            return values()[ordinal];
+        }
+
+        public static SuggestMode resolve(final String str) {
+            Objects.requireNonNull(str, "Input string is null");
+            return valueOf(str.toUpperCase(Locale.ROOT));
+        }
+
+        public abstract org.apache.lucene.search.spell.SuggestMode toLucene();
+    }
+
+    /** An enum representing the valid string edit distance algorithms for determining suggestions. */
+    public enum StringDistanceImpl implements Writeable<StringDistanceImpl> {
+        /** This is the default and is based on <code>damerau_levenshtein</code>, but highly optimized
+         * for comparing string distance for terms inside the index. */
+        INTERNAL {
+            @Override
+            public StringDistance toLucene() {
+                return DirectSpellChecker.INTERNAL_LEVENSHTEIN;
+            }
+        },
+        /** String distance algorithm based on Damerau-Levenshtein algorithm. */
+        DAMERAU_LEVENSHTEIN {
+            @Override
+            public StringDistance toLucene() {
+                return new LuceneLevenshteinDistance();
+            }
+        },
+        /** String distance algorithm based on Levenstein edit distance algorithm. */
+        LEVENSTEIN {
+            @Override
+            public StringDistance toLucene() {
+                return new LevensteinDistance();
+            }
+        },
+        /** String distance algorithm based on Jaro-Winkler algorithm. */
+        JAROWINKLER {
+            @Override
+            public StringDistance toLucene() {
+                return new JaroWinklerDistance();
+            }
+        },
+        /** String distance algorithm based on character n-grams. */
+        NGRAM {
+            @Override
+            public StringDistance toLucene() {
+                return new NGramDistance();
+            }
+        };
+
+        protected static StringDistanceImpl PROTOTYPE = INTERNAL;
+
+        @Override
+        public void writeTo(final StreamOutput out) throws IOException {
+            out.writeVInt(ordinal());
+        }
+
+        @Override
+        public StringDistanceImpl readFrom(final StreamInput in) throws IOException {
+            int ordinal = in.readVInt();
+            if (ordinal < 0 || ordinal >= values().length) {
+                throw new IOException("Unknown StringDistanceImpl ordinal [" + ordinal + "]");
+            }
+            return values()[ordinal];
+        }
+
+        public static StringDistanceImpl resolve(final String str) {
+            Objects.requireNonNull(str, "Input string is null");
+            final String distanceVal = str.toLowerCase(Locale.US);
+            switch (distanceVal) {
+                case "internal":
+                    return INTERNAL;
+                case "damerau_levenshtein":
+                case "damerauLevenshtein":
+                    return DAMERAU_LEVENSHTEIN;
+                case "levenstein":
+                    return LEVENSTEIN;
+                case "ngram":
+                    return NGRAM;
+                case "jarowinkler":
+                    return JAROWINKLER;
+                default: throw new IllegalArgumentException("Illegal distance option " + str);
+            }
+        }
+
+        public abstract StringDistance toLucene();
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java
index 4ff32d797ef..5102ef99014 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java
@@ -18,20 +18,25 @@
  */
 package org.elasticsearch.search.suggest.term;
 
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
-import org.elasticsearch.search.suggest.Suggester;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 
 final class TermSuggestionContext extends SuggestionContext {
 
     private final DirectSpellcheckerSettings settings = new DirectSpellcheckerSettings();
 
-    public TermSuggestionContext(Suggester<? extends TermSuggestionContext> suggester) {
-        super(suggester);
+    public TermSuggestionContext(QueryShardContext shardContext) {
+        super(TermSuggester.PROTOTYPE, shardContext);
     }
 
     public DirectSpellcheckerSettings getDirectSpellCheckerSettings() {
         return settings;
     }
 
-}
\ No newline at end of file
+    @Override
+    public String toString() {
+        return "SpellcheckerSettings" + settings + ", BaseSettings" + super.toString();
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index c6f189ea8a2..be2d6ccfea1 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -26,7 +26,6 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
@@ -49,6 +48,7 @@ import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -774,6 +774,33 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
         return false;
     }
 
+    /**
+     * Check if any of the indices to be closed are currently being restored from a snapshot and fail closing if such an index
+     * is found as closing an index that is being restored makes the index unusable (it cannot be recovered).
+     */
+    public static void checkIndexClosing(ClusterState currentState, Set<IndexMetaData> indices) {
+        RestoreInProgress restore = currentState.custom(RestoreInProgress.TYPE);
+        if (restore != null) {
+            Set<Index> indicesToFail = null;
+            for (RestoreInProgress.Entry entry : restore.entries()) {
+                for (ObjectObjectCursor<ShardId, RestoreInProgress.ShardRestoreStatus> shard : entry.shards()) {
+                    if (!shard.value.state().completed()) {
+                        IndexMetaData indexMetaData = currentState.metaData().index(shard.key.getIndex());
+                        if (indexMetaData != null && indices.contains(indexMetaData)) {
+                            if (indicesToFail == null) {
+                                indicesToFail = new HashSet<>();
+                            }
+                            indicesToFail.add(shard.key.getIndex());
+                        }
+                    }
+                }
+            }
+            if (indicesToFail != null) {
+                throw new IllegalArgumentException("Cannot close indices that are being restored: " + indicesToFail);
+            }
+        }
+    }
+
     /**
      * Adds restore completion listener
      * <p>
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
index 340a7f6ce83..4a15dbdac2e 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
@@ -23,13 +23,13 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.apache.lucene.index.IndexCommit;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
@@ -343,9 +343,9 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
                 indexShardRepository.snapshot(snapshotId, shardId, snapshotIndexCommit, snapshotStatus);
                 if (logger.isDebugEnabled()) {
                     StringBuilder sb = new StringBuilder();
-                    sb.append("snapshot (").append(snapshotId.getSnapshot()).append(") completed to ").append(indexShardRepository).append(", took [").append(TimeValue.timeValueMillis(snapshotStatus.time())).append("]\n");
                     sb.append("    index    : version [").append(snapshotStatus.indexVersion()).append("], number_of_files [").append(snapshotStatus.numberOfFiles()).append("] with total_size [").append(new ByteSizeValue(snapshotStatus.totalSize())).append("]\n");
-                    logger.debug(sb.toString());
+                    logger.debug("snapshot ({}) completed to {}, took [{}]\n{}", snapshotId.getSnapshot(), indexShardRepository,
+                        TimeValue.timeValueMillis(snapshotStatus.time()), sb);
                 }
             } finally {
                 indexShard.releaseSnapshot(snapshotIndexCommit);
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index 909fb4a1569..9305d5f941f 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -26,7 +26,6 @@ import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
@@ -44,6 +43,7 @@ import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.collect.Tuple;
@@ -51,6 +51,7 @@ import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
@@ -204,9 +205,9 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                 SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
                 if (snapshots == null || snapshots.entries().isEmpty()) {
                     // Store newSnapshot here to be processed in clusterStateProcessed
-                    List<String> indices = Arrays.asList(indexNameExpressionResolver.concreteIndices(currentState, request.indicesOptions(), request.indices()));
+                    List<String> indices = Arrays.asList(indexNameExpressionResolver.concreteIndexNames(currentState, request.indicesOptions(), request.indices()));
                     logger.trace("[{}][{}] creating snapshot for indices [{}]", request.repository(), request.name(), indices);
-                    newSnapshot = new SnapshotsInProgress.Entry(snapshotId, request.includeGlobalState(), State.INIT, indices, System.currentTimeMillis(), null);
+                    newSnapshot = new SnapshotsInProgress.Entry(snapshotId, request.includeGlobalState(), request.partial(), State.INIT, indices, System.currentTimeMillis(), null);
                     snapshots = new SnapshotsInProgress(newSnapshot);
                 } else {
                     // TODO: What should we do if a snapshot is already running?
@@ -228,7 +229,7 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                     threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new Runnable() {
                         @Override
                         public void run() {
-                            beginSnapshot(newState, newSnapshot, request.partial, listener);
+                            beginSnapshot(newState, newSnapshot, request.partial(), listener);
                         }
                     });
                 }
@@ -751,7 +752,7 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
         Set<String> closed = new HashSet<>();
         for (ObjectObjectCursor<ShardId, SnapshotsInProgress.ShardSnapshotStatus> entry : shards) {
             if (entry.value.state() == State.MISSING) {
-                if (metaData.hasIndex(entry.key.getIndex().getName()) && metaData.index(entry.key.getIndex()).getState() == IndexMetaData.State.CLOSE) {
+                if (metaData.hasIndex(entry.key.getIndex().getName()) && metaData.getIndexSafe(entry.key.getIndex()).getState() == IndexMetaData.State.CLOSE) {
                     closed.add(entry.key.getIndex().getName());
                 } else {
                     missing.add(entry.key.getIndex().getName());
@@ -841,7 +842,7 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
 
             @Override
             public void onFailure(String source, Throwable t) {
-                logger.warn("[{}][{}] failed to remove snapshot metadata", t, snapshotId);
+                logger.warn("[{}] failed to remove snapshot metadata", t, snapshotId);
             }
 
             @Override
@@ -1061,6 +1062,65 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
         return builder.build();
     }
 
+    /**
+     * Check if any of the indices to be deleted are currently being snapshotted. Fail as deleting an index that is being
+     * snapshotted (with partial == false) makes the snapshot fail.
+     */
+    public static void checkIndexDeletion(ClusterState currentState, Set<IndexMetaData> indices) {
+        Set<Index> indicesToFail = indicesToFailForCloseOrDeletion(currentState, indices);
+        if (indicesToFail != null) {
+            throw new IllegalArgumentException("Cannot delete indices that are being snapshotted: " + indicesToFail +
+                ". Try again after snapshot finishes or cancel the currently running snapshot.");
+        }
+    }
+
+    /**
+     * Check if any of the indices to be closed are currently being snapshotted. Fail as closing an index that is being
+     * snapshotted (with partial == false) makes the snapshot fail.
+     */
+    public static void checkIndexClosing(ClusterState currentState, Set<IndexMetaData> indices) {
+        Set<Index> indicesToFail = indicesToFailForCloseOrDeletion(currentState, indices);
+        if (indicesToFail != null) {
+            throw new IllegalArgumentException("Cannot close indices that are being snapshotted: " + indicesToFail +
+                ". Try again after snapshot finishes or cancel the currently running snapshot.");
+        }
+    }
+
+    private static Set<Index> indicesToFailForCloseOrDeletion(ClusterState currentState, Set<IndexMetaData> indices) {
+        SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
+        Set<Index> indicesToFail = null;
+        if (snapshots != null) {
+            for (final SnapshotsInProgress.Entry entry : snapshots.entries()) {
+                if (entry.partial() == false) {
+                    if (entry.state() == State.INIT) {
+                        for (String index : entry.indices()) {
+                            IndexMetaData indexMetaData = currentState.metaData().index(index);
+                            if (indexMetaData != null && indices.contains(indexMetaData)) {
+                                if (indicesToFail == null) {
+                                    indicesToFail = new HashSet<>();
+                                }
+                                indicesToFail.add(indexMetaData.getIndex());
+                            }
+                        }
+                    } else {
+                        for (ObjectObjectCursor<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shard : entry.shards()) {
+                            if (!shard.value.state().completed()) {
+                                IndexMetaData indexMetaData = currentState.metaData().index(shard.key.getIndex());
+                                if (indexMetaData != null && indices.contains(indexMetaData)) {
+                                    if (indicesToFail == null) {
+                                        indicesToFail = new HashSet<>();
+                                    }
+                                    indicesToFail.add(shard.key.getIndex());
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        return indicesToFail;
+    }
+
     /**
      * Adds snapshot completion listener
      *
@@ -1302,6 +1362,15 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
             return includeGlobalState;
         }
 
+        /**
+         * Returns true if partial snapshot should be allowed
+         *
+         * @return true if partial snapshot should be allowed
+         */
+        public boolean partial() {
+            return partial;
+        }
+
         /**
          * Returns master node timeout
          *
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskId.java b/core/src/main/java/org/elasticsearch/tasks/TaskId.java
index 5c5ad36cc17..d1f29e65226 100644
--- a/core/src/main/java/org/elasticsearch/tasks/TaskId.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskId.java
@@ -73,15 +73,15 @@ public final class TaskId implements Writeable<TaskId> {
     }
 
     public boolean isSet() {
-        return id == -1L;
+        return id != -1L;
     }
 
     @Override
     public String toString() {
         if (isSet()) {
-            return "unset";
-        } else {
             return nodeId + ":" + id;
+        } else {
+            return "unset";
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
index 0c785573c99..c881e1756e8 100644
--- a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
@@ -76,7 +76,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen
                 CancellableTaskHolder oldHolder = cancellableTasks.put(task.getId(), holder);
                 assert oldHolder == null;
                 // Check if this task was banned before we start it
-                if (task.getParentTaskId().isSet() == false && banedParents.isEmpty() == false) {
+                if (task.getParentTaskId().isSet() && banedParents.isEmpty() == false) {
                     String reason = banedParents.get(task.getParentTaskId());
                     if (reason != null) {
                         try {
@@ -241,7 +241,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen
                 CancellableTaskHolder holder = taskEntry.getValue();
                 CancellableTask task = holder.getTask();
                 TaskId parentTaskId = task.getParentTaskId();
-                if (parentTaskId.isSet() == false && lastDiscoveryNodes.nodeExists(parentTaskId.getNodeId()) == false) {
+                if (parentTaskId.isSet() && lastDiscoveryNodes.nodeExists(parentTaskId.getNodeId()) == false) {
                     if (task.cancelOnParentLeaving()) {
                         holder.cancel("Coordinating node [" + parentTaskId.getNodeId() + "] left the cluster");
                     }
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index 8b6f1198705..b223b477740 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -28,6 +28,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.unit.SizeValue;
@@ -81,8 +82,6 @@ public class ThreadPool extends AbstractComponent implements Closeable {
         public static final String INDEX = "index";
         public static final String BULK = "bulk";
         public static final String SEARCH = "search";
-        public static final String SUGGEST = "suggest";
-        public static final String PERCOLATE = "percolate";
         public static final String MANAGEMENT = "management";
         public static final String FLUSH = "flush";
         public static final String REFRESH = "refresh";
@@ -139,8 +138,6 @@ public class ThreadPool extends AbstractComponent implements Closeable {
         map.put(Names.INDEX, ThreadPoolType.FIXED);
         map.put(Names.BULK, ThreadPoolType.FIXED);
         map.put(Names.SEARCH, ThreadPoolType.FIXED);
-        map.put(Names.SUGGEST, ThreadPoolType.FIXED);
-        map.put(Names.PERCOLATE, ThreadPoolType.FIXED);
         map.put(Names.MANAGEMENT, ThreadPoolType.SCALING);
         map.put(Names.FLUSH, ThreadPoolType.SCALING);
         map.put(Names.REFRESH, ThreadPoolType.SCALING);
@@ -188,7 +185,8 @@ public class ThreadPool extends AbstractComponent implements Closeable {
         }
     }
 
-    public static final Setting<Settings> THREADPOOL_GROUP_SETTING = Setting.groupSetting("threadpool.", true, Setting.Scope.CLUSTER);
+    public static final Setting<Settings> THREADPOOL_GROUP_SETTING =
+        Setting.groupSetting("threadpool.", Property.Dynamic, Property.NodeScope);
 
     private volatile Map<String, ExecutorHolder> executors;
 
@@ -227,8 +225,6 @@ public class ThreadPool extends AbstractComponent implements Closeable {
         add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.BULK).size(availableProcessors).queueSize(50));
         add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.GET).size(availableProcessors).queueSize(1000));
         add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.SEARCH).size(((availableProcessors * 3) / 2) + 1).queueSize(1000));
-        add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.SUGGEST).size(availableProcessors).queueSize(1000));
-        add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.PERCOLATE).size(availableProcessors).queueSize(1000));
         add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.MANAGEMENT).size(5).keepAlive("5m"));
         // no queue as this means clients will need to handle rejections on listener queue even if the operation succeeded
         // the assumption here is that the listeners should be very lightweight on the listeners side
@@ -339,10 +335,18 @@ public class ThreadPool extends AbstractComponent implements Closeable {
         return new ThreadPoolStats(stats);
     }
 
+    /**
+     * Get the generic executor. This executor's {@link Executor#execute(Runnable)} method will run the Runnable it is given in
+     * the {@link ThreadContext} of the thread that queues it.
+     */
     public Executor generic() {
         return executor(Names.GENERIC);
     }
 
+    /**
+     * Get the executor with the given name. This executor's {@link Executor#execute(Runnable)} method will run the Runnable it is given in
+     * the {@link ThreadContext} of the thread that queues it.
+     */
     public Executor executor(String name) {
         Executor executor = executors.get(name).executor();
         if (executor == null) {
@@ -355,10 +359,31 @@ public class ThreadPool extends AbstractComponent implements Closeable {
         return this.scheduler;
     }
 
+    /**
+     * Schedules a periodic action that always runs on the scheduler thread.
+     *
+     * @param command the action to take
+     * @param interval the delay interval
+     * @return a ScheduledFuture who's get will return when the task is complete and throw an exception if it is canceled
+     */
     public ScheduledFuture<?> scheduleWithFixedDelay(Runnable command, TimeValue interval) {
         return scheduler.scheduleWithFixedDelay(new LoggingRunnable(command), interval.millis(), interval.millis(), TimeUnit.MILLISECONDS);
     }
 
+    /**
+     * Schedules a one-shot command to run after a given delay. The command is not run in the context of the calling thread. To preserve the
+     * context of the calling thread you may call <code>threadPool.getThreadContext().preserveContext</code> on the runnable before passing
+     * it to this method.
+     *
+     * @param delay delay before the task executes
+     * @param name the name of the thread pool on which to execute this task. SAME means "execute on the scheduler thread" which changes the
+     *        meaning of the ScheduledFuture returned by this method. In that case the ScheduledFuture will complete only when the command
+     *        completes.
+     * @param command the command to run
+     * @return a ScheduledFuture who's get will return when the task is has been added to its target thread pool and throw an exception if
+     *         the task is canceled before it was added to its target thread pool. Once the task has been added to its target thread pool
+     *         the ScheduledFuture will cannot interact with it.
+     */
     public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
         if (!Names.SAME.equals(name)) {
             command = new ThreadedRunnable(command, executor(name));
diff --git a/core/src/main/java/org/elasticsearch/transport/Transport.java b/core/src/main/java/org/elasticsearch/transport/Transport.java
index c930773f39c..532c9d99ace 100644
--- a/core/src/main/java/org/elasticsearch/transport/Transport.java
+++ b/core/src/main/java/org/elasticsearch/transport/Transport.java
@@ -22,6 +22,7 @@ package org.elasticsearch.transport;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 
@@ -35,7 +36,7 @@ import java.util.Map;
 public interface Transport extends LifecycleComponent<Transport> {
 
 
-    Setting<Boolean> TRANSPORT_TCP_COMPRESS = Setting.boolSetting("transport.tcp.compress", false, false, Setting.Scope.CLUSTER);
+    Setting<Boolean> TRANSPORT_TCP_COMPRESS = Setting.boolSetting("transport.tcp.compress", false, Property.NodeScope);
 
     void transportServiceAdapter(TransportServiceAdapter service);
 
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportService.java b/core/src/main/java/org/elasticsearch/transport/TransportService.java
index 7884ed04af1..2d804bfc786 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -20,20 +20,18 @@
 package org.elasticsearch.transport;
 
 import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction;
-import org.elasticsearch.action.support.replication.ReplicationTask;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.metrics.MeanMetric;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Scope;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -43,7 +41,6 @@ import org.elasticsearch.common.util.concurrent.ConcurrentMapLong;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -98,10 +95,11 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
 
     // tracer log
 
-    public static final Setting<List<String>> TRACE_LOG_INCLUDE_SETTING = listSetting("transport.tracer.include", emptyList(),
-            Function.identity(), true, Scope.CLUSTER);
-    public static final Setting<List<String>> TRACE_LOG_EXCLUDE_SETTING = listSetting("transport.tracer.exclude",
-            Arrays.asList("internal:discovery/zen/fd*", TransportLivenessAction.NAME), Function.identity(), true, Scope.CLUSTER);
+    public static final Setting<List<String>> TRACE_LOG_INCLUDE_SETTING =
+        listSetting("transport.tracer.include", emptyList(), Function.identity(), Property.Dynamic, Property.NodeScope);
+    public static final Setting<List<String>> TRACE_LOG_EXCLUDE_SETTING =
+        listSetting("transport.tracer.exclude", Arrays.asList("internal:discovery/zen/fd*", TransportLivenessAction.NAME),
+            Function.identity(), Property.Dynamic, Property.NodeScope);
 
     private final ESLogger tracerLog;
 
@@ -112,11 +110,11 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
     volatile DiscoveryNode localNode = null;
 
     public TransportService(Transport transport, ThreadPool threadPool) {
-        this(EMPTY_SETTINGS, transport, threadPool, new NamedWriteableRegistry());
+        this(EMPTY_SETTINGS, transport, threadPool);
     }
 
     @Inject
-    public TransportService(Settings settings, Transport transport, ThreadPool threadPool, NamedWriteableRegistry namedWriteableRegistry) {
+    public TransportService(Settings settings, Transport transport, ThreadPool threadPool) {
         super(settings);
         this.transport = transport;
         this.threadPool = threadPool;
@@ -125,7 +123,6 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
         tracerLog = Loggers.getLogger(logger, ".tracer");
         adapter = createAdapter();
         taskManager = createTaskManager();
-        namedWriteableRegistry.registerPrototype(Task.Status.class, ReplicationTask.Status.PROTOTYPE);
     }
 
     /**
@@ -380,7 +377,7 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
                         try {
                             channel.sendResponse(e);
                         } catch (Throwable e1) {
-                            logger.warn("failed to notify channel of error message for action [" + action + "]", e1);
+                            logger.warn("failed to notify channel of error message for action [{}]", e1, action);
                             logger.warn("actual exception", e);
                         }
                     }
@@ -391,7 +388,7 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
             try {
                 channel.sendResponse(e);
             } catch (Throwable e1) {
-                logger.warn("failed to notify channel of error message for action [" + action + "]", e1);
+                logger.warn("failed to notify channel of error message for action [{}]", e1, action);
                 logger.warn("actual exception", e1);
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportSettings.java b/core/src/main/java/org/elasticsearch/transport/TransportSettings.java
index e02dcc412ed..e36d4e403c4 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportSettings.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportSettings.java
@@ -19,10 +19,11 @@
 package org.elasticsearch.transport;
 
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Scope;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 import java.util.List;
+import java.util.function.Function;
 
 import static java.util.Collections.emptyList;
 import static org.elasticsearch.common.settings.Setting.groupSetting;
@@ -35,13 +36,19 @@ import static org.elasticsearch.common.settings.Setting.listSetting;
  */
 final public class TransportSettings {
 
-    public static final Setting<List<String>> HOST = listSetting("transport.host", emptyList(), s -> s, false, Scope.CLUSTER);
-    public static final Setting<List<String>> PUBLISH_HOST = listSetting("transport.publish_host", HOST, s -> s, false, Scope.CLUSTER);
-    public static final Setting<List<String>> BIND_HOST = listSetting("transport.bind_host", HOST, s -> s, false, Scope.CLUSTER);
-    public static final Setting<String> PORT = new Setting<>("transport.tcp.port", "9300-9400", s -> s, false, Scope.CLUSTER);
-    public static final Setting<Integer> PUBLISH_PORT = intSetting("transport.publish_port", -1, -1, false, Scope.CLUSTER);
+    public static final Setting<List<String>> HOST =
+        listSetting("transport.host", emptyList(), Function.identity(), Property.NodeScope);
+    public static final Setting<List<String>> PUBLISH_HOST =
+        listSetting("transport.publish_host", HOST, Function.identity(), Property.NodeScope);
+    public static final Setting<List<String>> BIND_HOST =
+        listSetting("transport.bind_host", HOST, Function.identity(), Property.NodeScope);
+    public static final Setting<String> PORT =
+        new Setting<>("transport.tcp.port", "9300-9400", Function.identity(), Property.NodeScope);
+    public static final Setting<Integer> PUBLISH_PORT =
+        intSetting("transport.publish_port", -1, -1, Property.NodeScope);
     public static final String DEFAULT_PROFILE = "default";
-    public static final Setting<Settings> TRANSPORT_PROFILES_SETTING = groupSetting("transport.profiles.", true, Scope.CLUSTER);
+    public static final Setting<Settings> TRANSPORT_PROFILES_SETTING =
+        groupSetting("transport.profiles.", Property.Dynamic, Property.NodeScope);
 
     private TransportSettings() {
 
diff --git a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
index faef71998a9..a7783148ef3 100644
--- a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
@@ -272,7 +272,7 @@ public class LocalTransport extends AbstractLifecycleComponent<Transport> implem
                     handleException(handler, new RemoteTransportException(nodeName(), localAddress, action, e));
                 }
             } else {
-                logger.warn("Failed to receive message for action [" + action + "]", e);
+                logger.warn("Failed to receive message for action [{}]", e, action);
             }
         }
     }
@@ -314,7 +314,7 @@ public class LocalTransport extends AbstractLifecycleComponent<Transport> implem
                             try {
                                 transportChannel.sendResponse(e);
                             } catch (Throwable e1) {
-                                logger.warn("Failed to send error message back to client for action [" + action + "]", e1);
+                                logger.warn("Failed to send error message back to client for action [{}]", e1, action);
                                 logger.warn("Actual Exception", e);
                             }
                         }
@@ -325,7 +325,7 @@ public class LocalTransport extends AbstractLifecycleComponent<Transport> implem
             try {
                 transportChannel.sendResponse(e);
             } catch (Throwable e1) {
-                logger.warn("Failed to send error message back to client for action [" + action + "]", e);
+                logger.warn("Failed to send error message back to client for action [{}]", e, action);
                 logger.warn("Actual Exception", e1);
             }
 
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java b/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java
index e4dbbfa73af..9eef4401144 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java
@@ -127,6 +127,10 @@ public class MessageChannelHandler extends SimpleChannelUpstreamHandler {
                 }
                 streamIn = compressor.streamInput(streamIn);
             }
+            if (version.onOrAfter(Version.CURRENT.minimumCompatibilityVersion()) == false || version.major != Version.CURRENT.major) {
+                throw new IllegalStateException("Received message from unsupported version: [" + version
+                    + "] minimal compatible version is: [" +Version.CURRENT.minimumCompatibilityVersion() + "]");
+            }
             streamIn.setVersion(version);
             if (TransportStatus.isRequest(status)) {
                 threadContext.readHeaders(streamIn);
@@ -274,7 +278,7 @@ public class MessageChannelHandler extends SimpleChannelUpstreamHandler {
             try {
                 transportChannel.sendResponse(e);
             } catch (IOException e1) {
-                logger.warn("Failed to send error message back to client for action [" + action + "]", e);
+                logger.warn("Failed to send error message back to client for action [{}]", e, action);
                 logger.warn("Actual Exception", e1);
             }
         }
@@ -336,7 +340,7 @@ public class MessageChannelHandler extends SimpleChannelUpstreamHandler {
                 try {
                     transportChannel.sendResponse(e);
                 } catch (Throwable e1) {
-                    logger.warn("Failed to send error message back to client for action [" + reg.getAction() + "]", e1);
+                    logger.warn("Failed to send error message back to client for action [{}]", e1, reg.getAction());
                     logger.warn("Actual Exception", e);
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyInternalESLogger.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyInternalESLogger.java
index ed92aa261db..2a1fc3226a4 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyInternalESLogger.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyInternalESLogger.java
@@ -19,12 +19,14 @@
 
 package org.elasticsearch.transport.netty;
 
+import org.elasticsearch.common.SuppressLoggerChecks;
 import org.elasticsearch.common.logging.ESLogger;
 import org.jboss.netty.logging.AbstractInternalLogger;
 
 /**
  *
  */
+@SuppressLoggerChecks(reason = "safely delegates to logger")
 public class NettyInternalESLogger extends AbstractInternalLogger {
 
     private final ESLogger logger;
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
index dc9dd70ab8d..da629a0d47f 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
@@ -33,6 +33,7 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.math.MathUtils;
 import org.elasticsearch.common.metrics.CounterMetric;
@@ -44,7 +45,7 @@ import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.network.NetworkService.TcpSettings;
 import org.elasticsearch.common.network.NetworkUtils;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Scope;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
@@ -149,40 +150,45 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
     public static final String TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX = "transport_client_worker";
     public static final String TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX = "transport_client_boss";
 
-    public static final Setting<Integer> WORKER_COUNT = new Setting<>("transport.netty.worker_count",
+    public static final Setting<Integer> WORKER_COUNT =
+        new Setting<>("transport.netty.worker_count",
             (s) -> Integer.toString(EsExecutors.boundedNumberOfProcessors(s) * 2),
-            (s) -> Setting.parseInt(s, 1, "transport.netty.worker_count"), false, Setting.Scope.CLUSTER);
-    public static final Setting<Integer> CONNECTIONS_PER_NODE_RECOVERY = intSetting("transport.connections_per_node.recovery", 2, 1, false,
-            Scope.CLUSTER);
-    public static final Setting<Integer> CONNECTIONS_PER_NODE_BULK = intSetting("transport.connections_per_node.bulk", 3, 1, false,
-            Scope.CLUSTER);
-    public static final Setting<Integer> CONNECTIONS_PER_NODE_REG = intSetting("transport.connections_per_node.reg", 6, 1, false,
-            Scope.CLUSTER);
-    public static final Setting<Integer> CONNECTIONS_PER_NODE_STATE = intSetting("transport.connections_per_node.state", 1, 1, false,
-            Scope.CLUSTER);
-    public static final Setting<Integer> CONNECTIONS_PER_NODE_PING = intSetting("transport.connections_per_node.ping", 1, 1, false,
-            Scope.CLUSTER);
+            (s) -> Setting.parseInt(s, 1, "transport.netty.worker_count"), Property.NodeScope);
+    public static final Setting<Integer> CONNECTIONS_PER_NODE_RECOVERY =
+        intSetting("transport.connections_per_node.recovery", 2, 1, Property.NodeScope);
+    public static final Setting<Integer> CONNECTIONS_PER_NODE_BULK =
+        intSetting("transport.connections_per_node.bulk", 3, 1, Property.NodeScope);
+    public static final Setting<Integer> CONNECTIONS_PER_NODE_REG =
+        intSetting("transport.connections_per_node.reg", 6, 1, Property.NodeScope);
+    public static final Setting<Integer> CONNECTIONS_PER_NODE_STATE =
+        intSetting("transport.connections_per_node.state", 1, 1, Property.NodeScope);
+    public static final Setting<Integer> CONNECTIONS_PER_NODE_PING =
+        intSetting("transport.connections_per_node.ping", 1, 1, Property.NodeScope);
     // the scheduled internal ping interval setting, defaults to disabled (-1)
-    public static final Setting<TimeValue> PING_SCHEDULE = timeSetting("transport.ping_schedule", TimeValue.timeValueSeconds(-1), false,
-            Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> TCP_BLOCKING_CLIENT = boolSetting("transport.tcp.blocking_client", TcpSettings.TCP_BLOCKING_CLIENT,
-            false, Setting.Scope.CLUSTER);
-    public static final Setting<TimeValue> TCP_CONNECT_TIMEOUT = timeSetting("transport.tcp.connect_timeout",
-            TcpSettings.TCP_CONNECT_TIMEOUT, false, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> TCP_NO_DELAY = boolSetting("transport.tcp_no_delay", TcpSettings.TCP_NO_DELAY, false,
-            Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> TCP_KEEP_ALIVE = boolSetting("transport.tcp.keep_alive", TcpSettings.TCP_KEEP_ALIVE, false,
-            Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> TCP_BLOCKING_SERVER = boolSetting("transport.tcp.blocking_server", TcpSettings.TCP_BLOCKING_SERVER,
-            false, Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> TCP_REUSE_ADDRESS = boolSetting("transport.tcp.reuse_address", TcpSettings.TCP_REUSE_ADDRESS,
-            false, Setting.Scope.CLUSTER);
+    public static final Setting<TimeValue> PING_SCHEDULE =
+        timeSetting("transport.ping_schedule", TimeValue.timeValueSeconds(-1), Property.NodeScope);
+    public static final Setting<Boolean> TCP_BLOCKING_CLIENT =
+        boolSetting("transport.tcp.blocking_client", TcpSettings.TCP_BLOCKING_CLIENT, Property.NodeScope);
+    public static final Setting<TimeValue> TCP_CONNECT_TIMEOUT =
+        timeSetting("transport.tcp.connect_timeout", TcpSettings.TCP_CONNECT_TIMEOUT, Property.NodeScope);
+    public static final Setting<Boolean> TCP_NO_DELAY =
+        boolSetting("transport.tcp_no_delay", TcpSettings.TCP_NO_DELAY, Property.NodeScope);
+    public static final Setting<Boolean> TCP_KEEP_ALIVE =
+        boolSetting("transport.tcp.keep_alive", TcpSettings.TCP_KEEP_ALIVE, Property.NodeScope);
+    public static final Setting<Boolean> TCP_BLOCKING_SERVER =
+        boolSetting("transport.tcp.blocking_server", TcpSettings.TCP_BLOCKING_SERVER, Property.NodeScope);
+    public static final Setting<Boolean> TCP_REUSE_ADDRESS =
+        boolSetting("transport.tcp.reuse_address", TcpSettings.TCP_REUSE_ADDRESS, Property.NodeScope);
 
-    public static final Setting<ByteSizeValue> TCP_SEND_BUFFER_SIZE = Setting.byteSizeSetting("transport.tcp.send_buffer_size", TcpSettings.TCP_SEND_BUFFER_SIZE, false, Setting.Scope.CLUSTER);
-    public static final Setting<ByteSizeValue> TCP_RECEIVE_BUFFER_SIZE = Setting.byteSizeSetting("transport.tcp.receive_buffer_size", TcpSettings.TCP_RECEIVE_BUFFER_SIZE, false, Setting.Scope.CLUSTER);
+    public static final Setting<ByteSizeValue> TCP_SEND_BUFFER_SIZE =
+        Setting.byteSizeSetting("transport.tcp.send_buffer_size", TcpSettings.TCP_SEND_BUFFER_SIZE, Property.NodeScope);
+    public static final Setting<ByteSizeValue> TCP_RECEIVE_BUFFER_SIZE =
+        Setting.byteSizeSetting("transport.tcp.receive_buffer_size", TcpSettings.TCP_RECEIVE_BUFFER_SIZE, Property.NodeScope);
 
-    public static final Setting<ByteSizeValue> NETTY_MAX_CUMULATION_BUFFER_CAPACITY = Setting.byteSizeSetting("transport.netty.max_cumulation_buffer_capacity", new ByteSizeValue(-1), false, Setting.Scope.CLUSTER);
-    public static final Setting<Integer> NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS = Setting.intSetting("transport.netty.max_composite_buffer_components", -1, -1, false, Setting.Scope.CLUSTER);
+    public static final Setting<ByteSizeValue> NETTY_MAX_CUMULATION_BUFFER_CAPACITY =
+        Setting.byteSizeSetting("transport.netty.max_cumulation_buffer_capacity", new ByteSizeValue(-1), Property.NodeScope);
+    public static final Setting<Integer> NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS =
+        Setting.intSetting("transport.netty.max_composite_buffer_components", -1, -1, Property.NodeScope);
 
     // See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one
     public static final Setting<ByteSizeValue> NETTY_RECEIVE_PREDICTOR_SIZE = Setting.byteSizeSetting(
@@ -195,12 +201,13 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
                     defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024));
                 }
                 return new ByteSizeValue(defaultReceiverPredictor).toString();
-            }, false, Setting.Scope.CLUSTER);
-    public static final Setting<ByteSizeValue> NETTY_RECEIVE_PREDICTOR_MIN = byteSizeSetting("transport.netty.receive_predictor_min",
-            NETTY_RECEIVE_PREDICTOR_SIZE, false, Scope.CLUSTER);
-    public static final Setting<ByteSizeValue> NETTY_RECEIVE_PREDICTOR_MAX = byteSizeSetting("transport.netty.receive_predictor_max",
-            NETTY_RECEIVE_PREDICTOR_SIZE, false, Scope.CLUSTER);
-    public static final Setting<Integer> NETTY_BOSS_COUNT = intSetting("transport.netty.boss_count", 1, 1, false, Scope.CLUSTER);
+            }, Property.NodeScope);
+    public static final Setting<ByteSizeValue> NETTY_RECEIVE_PREDICTOR_MIN =
+        byteSizeSetting("transport.netty.receive_predictor_min", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope);
+    public static final Setting<ByteSizeValue> NETTY_RECEIVE_PREDICTOR_MAX =
+        byteSizeSetting("transport.netty.receive_predictor_max", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope);
+    public static final Setting<Integer> NETTY_BOSS_COUNT =
+        intSetting("transport.netty.boss_count", 1, 1, Property.NodeScope);
 
     protected final NetworkService networkService;
     protected final Version version;
@@ -943,8 +950,8 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
         }
         globalLock.readLock().lock();
         try {
-            connectionLock.acquire(node.id());
-            try {
+
+            try (Releasable ignored = connectionLock.acquire(node.id())) {
                 if (!lifecycle.started()) {
                     throw new IllegalStateException("can't add nodes to a stopped transport");
                 }
@@ -979,8 +986,6 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
                 } catch (Exception e) {
                     throw new ConnectTransportException(node, "general node connection failure", e);
                 }
-            } finally {
-                connectionLock.release(node.id());
             }
         } finally {
             globalLock.readLock().unlock();
@@ -1103,8 +1108,8 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
 
     @Override
     public void disconnectFromNode(DiscoveryNode node) {
-        connectionLock.acquire(node.id());
-        try {
+
+        try (Releasable ignored = connectionLock.acquire(node.id())) {
             NodeChannels nodeChannels = connectedNodes.remove(node);
             if (nodeChannels != null) {
                 try {
@@ -1115,8 +1120,6 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
                     transportServiceAdapter.raiseNodeDisconnected(node);
                 }
             }
-        } finally {
-            connectionLock.release(node.id());
         }
     }
 
@@ -1128,8 +1131,7 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
         // check outside of the lock
         NodeChannels nodeChannels = connectedNodes.get(node);
         if (nodeChannels != null && nodeChannels.hasChannel(channel)) {
-            connectionLock.acquire(node.id());
-            try {
+            try (Releasable ignored = connectionLock.acquire(node.id())) {
                 nodeChannels = connectedNodes.get(node);
                 // check again within the connection lock, if its still applicable to remove it
                 if (nodeChannels != null && nodeChannels.hasChannel(channel)) {
@@ -1143,8 +1145,6 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
                     }
                     return true;
                 }
-            } finally {
-                connectionLock.release(node.id());
             }
         }
         return false;
@@ -1385,9 +1385,9 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
         @Override
         public void onFailure(Throwable t) {
             if (lifecycle.stoppedOrClosed()) {
-                logger.trace("[{}] failed to send ping transport message", t);
+                logger.trace("failed to send ping transport message", t);
             } else {
-                logger.warn("[{}] failed to send ping transport message", t);
+                logger.warn("failed to send ping transport message", t);
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/tribe/TribeService.java b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
index bf66cce1b9e..f46c6034626 100644
--- a/core/src/main/java/org/elasticsearch/tribe/TribeService.java
+++ b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
@@ -23,7 +23,6 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateTaskConfig;
@@ -37,12 +36,16 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.set.Sets;
@@ -52,7 +55,9 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.transport.TransportSettings;
 
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -119,7 +124,7 @@ public class TribeService extends AbstractLifecycleComponent<TribeService> {
     }
 
     // internal settings only
-    public static final Setting<String> TRIBE_NAME_SETTING = Setting.simpleString("tribe.name", false, Setting.Scope.CLUSTER);
+    public static final Setting<String> TRIBE_NAME_SETTING = Setting.simpleString("tribe.name", Property.NodeScope);
     private final ClusterService clusterService;
     private final String[] blockIndicesWrite;
     private final String[] blockIndicesRead;
@@ -138,22 +143,31 @@ public class TribeService extends AbstractLifecycleComponent<TribeService> {
                 throw new IllegalArgumentException(
                         "Invalid value for [tribe.on_conflict] must be either [any, drop or start with prefer_] but was: [" + s + "]");
         }
-    }, false, Setting.Scope.CLUSTER);
+    }, Property.NodeScope);
 
-    public static final Setting<Boolean> BLOCKS_METADATA_SETTING = Setting.boolSetting("tribe.blocks.metadata", false, false,
-            Setting.Scope.CLUSTER);
-    public static final Setting<Boolean> BLOCKS_WRITE_SETTING = Setting.boolSetting("tribe.blocks.write", false, false,
-            Setting.Scope.CLUSTER);
-    public static final Setting<List<String>> BLOCKS_WRITE_INDICES_SETTING = Setting.listSetting("tribe.blocks.write.indices",
-            Collections.emptyList(), Function.identity(), false, Setting.Scope.CLUSTER);
-    public static final Setting<List<String>> BLOCKS_READ_INDICES_SETTING = Setting.listSetting("tribe.blocks.read.indices",
-            Collections.emptyList(), Function.identity(), false, Setting.Scope.CLUSTER);
-    public static final Setting<List<String>> BLOCKS_METADATA_INDICES_SETTING = Setting.listSetting("tribe.blocks.metadata.indices",
-            Collections.emptyList(), Function.identity(), false, Setting.Scope.CLUSTER);
+    public static final Setting<Boolean> BLOCKS_METADATA_SETTING =
+        Setting.boolSetting("tribe.blocks.metadata", false, Property.NodeScope);
+    public static final Setting<Boolean> BLOCKS_WRITE_SETTING =
+        Setting.boolSetting("tribe.blocks.write", false, Property.NodeScope);
+    public static final Setting<List<String>> BLOCKS_WRITE_INDICES_SETTING =
+        Setting.listSetting("tribe.blocks.write.indices", Collections.emptyList(), Function.identity(), Property.NodeScope);
+    public static final Setting<List<String>> BLOCKS_READ_INDICES_SETTING =
+        Setting.listSetting("tribe.blocks.read.indices", Collections.emptyList(), Function.identity(), Property.NodeScope);
+    public static final Setting<List<String>> BLOCKS_METADATA_INDICES_SETTING =
+        Setting.listSetting("tribe.blocks.metadata.indices", Collections.emptyList(), Function.identity(), Property.NodeScope);
 
     public static final Set<String> TRIBE_SETTING_KEYS = Sets.newHashSet(TRIBE_NAME_SETTING.getKey(), ON_CONFLICT_SETTING.getKey(),
             BLOCKS_METADATA_INDICES_SETTING.getKey(), BLOCKS_METADATA_SETTING.getKey(), BLOCKS_READ_INDICES_SETTING.getKey(), BLOCKS_WRITE_INDICES_SETTING.getKey(), BLOCKS_WRITE_SETTING.getKey());
 
+    // these settings should be passed through to each tribe client, if they are not set explicitly
+    private static final List<Setting<?>> PASS_THROUGH_SETTINGS = Arrays.asList(
+        NetworkService.GLOBAL_NETWORK_HOST_SETTING,
+        NetworkService.GLOBAL_NETWORK_BINDHOST_SETTING,
+        NetworkService.GLOBAL_NETWORK_PUBLISHHOST_SETTING,
+        TransportSettings.HOST,
+        TransportSettings.BIND_HOST,
+        TransportSettings.PUBLISH_HOST
+    );
     private final String onConflict;
     private final Set<String> droppedIndices = ConcurrentCollections.newConcurrentSet();
 
@@ -167,18 +181,8 @@ public class TribeService extends AbstractLifecycleComponent<TribeService> {
         nodesSettings.remove("blocks"); // remove prefix settings that don't indicate a client
         nodesSettings.remove("on_conflict"); // remove prefix settings that don't indicate a client
         for (Map.Entry<String, Settings> entry : nodesSettings.entrySet()) {
-            Settings.Builder sb = Settings.builder().put(entry.getValue());
-            sb.put("node.name", settings.get("node.name") + "/" + entry.getKey());
-            sb.put(Environment.PATH_HOME_SETTING.getKey(), Environment.PATH_HOME_SETTING.get(settings)); // pass through ES home dir
-            if (Environment.PATH_CONF_SETTING.exists(settings)) {
-                sb.put(Environment.PATH_CONF_SETTING.getKey(), Environment.PATH_CONF_SETTING.get(settings));
-            }
-            sb.put(TRIBE_NAME_SETTING.getKey(), entry.getKey());
-            if (sb.get("http.enabled") == null) {
-                sb.put("http.enabled", false);
-            }
-            sb.put(Node.NODE_CLIENT_SETTING.getKey(), true);
-            nodes.add(new TribeClientNode(sb.build()));
+            Settings clientSettings = buildClientSettings(entry.getKey(), settings, entry.getValue());
+            nodes.add(new TribeClientNode(clientSettings));
         }
 
         this.blockIndicesMetadata = BLOCKS_METADATA_INDICES_SETTING.get(settings).toArray(Strings.EMPTY_ARRAY);
@@ -197,6 +201,46 @@ public class TribeService extends AbstractLifecycleComponent<TribeService> {
         this.onConflict = ON_CONFLICT_SETTING.get(settings);
     }
 
+    // pkg private for testing
+    /**
+     * Builds node settings for a tribe client node from the tribe node's global settings,
+     * combined with tribe specific settings.
+     */
+    static Settings buildClientSettings(String tribeName, Settings globalSettings, Settings tribeSettings) {
+        for (String tribeKey : tribeSettings.getAsMap().keySet()) {
+            if (tribeKey.startsWith("path.")) {
+                throw new IllegalArgumentException("Setting [" + tribeKey + "] not allowed in tribe client [" + tribeName + "]");
+            }
+        }
+        Settings.Builder sb = Settings.builder().put(tribeSettings);
+        sb.put("node.name", globalSettings.get("node.name") + "/" + tribeName);
+        sb.put(Environment.PATH_HOME_SETTING.getKey(), Environment.PATH_HOME_SETTING.get(globalSettings)); // pass through ES home dir
+        if (Environment.PATH_CONF_SETTING.exists(globalSettings)) {
+            sb.put(Environment.PATH_CONF_SETTING.getKey(), Environment.PATH_CONF_SETTING.get(globalSettings));
+        }
+        if (Environment.PATH_PLUGINS_SETTING.exists(globalSettings)) {
+            sb.put(Environment.PATH_PLUGINS_SETTING.getKey(), Environment.PATH_PLUGINS_SETTING.get(globalSettings));
+        }
+        if (Environment.PATH_LOGS_SETTING.exists(globalSettings)) {
+            sb.put(Environment.PATH_LOGS_SETTING.getKey(), Environment.PATH_LOGS_SETTING.get(globalSettings));
+        }
+        if (Environment.PATH_SCRIPTS_SETTING.exists(globalSettings)) {
+            sb.put(Environment.PATH_SCRIPTS_SETTING.getKey(), Environment.PATH_SCRIPTS_SETTING.get(globalSettings));
+        }
+        for (Setting<?> passthrough : PASS_THROUGH_SETTINGS) {
+            if (passthrough.exists(tribeSettings) == false && passthrough.exists(globalSettings)) {
+                sb.put(passthrough.getKey(), globalSettings.get(passthrough.getKey()));
+            }
+        }
+        sb.put(TRIBE_NAME_SETTING.getKey(), tribeName);
+        if (sb.get(NetworkModule.HTTP_ENABLED.getKey()) == null) {
+            sb.put(NetworkModule.HTTP_ENABLED.getKey(), false);
+        }
+        sb.put(Node.NODE_CLIENT_SETTING.getKey(), true);
+        return sb.build();
+    }
+
+
     @Override
     protected void doStart() {
         if (nodes.isEmpty() == false) {
@@ -218,7 +262,7 @@ public class TribeService extends AbstractLifecycleComponent<TribeService> {
                     try {
                         otherNode.close();
                     } catch (Throwable t) {
-                        logger.warn("failed to close node {} on failed start", otherNode, t);
+                        logger.warn("failed to close node {} on failed start", t, otherNode);
                     }
                 }
                 if (e instanceof RuntimeException) {
@@ -360,9 +404,11 @@ public class TribeService extends AbstractLifecycleComponent<TribeService> {
                 if (table == null) {
                     continue;
                 }
-                final IndexMetaData indexMetaData = currentState.metaData().index(tribeIndex.getIndex());
+                //NOTE: we have to use the index name here since UUID are different even if the name is the same
+                final String indexName = tribeIndex.getIndex().getName();
+                final IndexMetaData indexMetaData = currentState.metaData().index(indexName);
                 if (indexMetaData == null) {
-                    if (!droppedIndices.contains(tribeIndex.getIndex().getName())) {
+                    if (!droppedIndices.contains(indexName)) {
                         // a new index, add it, and add the tribe name as a setting
                         clusterStateChanged = true;
                         logger.info("[{}] adding index {}", tribeName, tribeIndex.getIndex());
@@ -380,7 +426,7 @@ public class TribeService extends AbstractLifecycleComponent<TribeService> {
                             logger.info("[{}] dropping index {} due to conflict with [{}]", tribeName, tribeIndex.getIndex(),
                                     existingFromTribe);
                             removeIndex(blocks, metaData, routingTable, tribeIndex);
-                            droppedIndices.add(tribeIndex.getIndex().getName());
+                            droppedIndices.add(indexName);
                         } else if (onConflict.startsWith(ON_CONFLICT_PREFER)) {
                             // on conflict, prefer a tribe...
                             String preferredTribeName = onConflict.substring(ON_CONFLICT_PREFER.length());
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/elasticsearch-start.help b/core/src/main/resources/org/elasticsearch/bootstrap/elasticsearch-start.help
deleted file mode 100644
index 9b27a8dd390..00000000000
--- a/core/src/main/resources/org/elasticsearch/bootstrap/elasticsearch-start.help
+++ /dev/null
@@ -1,28 +0,0 @@
-NAME
-
-    start - Start Elasticsearch
-
-SYNOPSIS
-
-    elasticsearch start
-
-DESCRIPTION
-
-    This command starts Elasticsearch. You can configure it to run in the foreground, write a pid file
-    and configure arbitrary options that override file-based configuration.
-
-OPTIONS
-
-    -h,--help                    Shows this message
-
-    -p,--pidfile <pidfile>       Creates a pid file in the specified path on start
-
-    -d,--daemonize               Starts Elasticsearch in the background
-
-    -Dproperty=value             Configures an Elasticsearch specific property, like -Dnetwork.host=127.0.0.1
-
-    --property=value             Configures an elasticsearch specific property, like --network.host 127.0.0.1
-    --property value
-
-    NOTE: The -d, -p, and -D arguments must appear before any --property arguments.
-
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/elasticsearch-version.help b/core/src/main/resources/org/elasticsearch/bootstrap/elasticsearch-version.help
deleted file mode 100644
index 00f2a33401c..00000000000
--- a/core/src/main/resources/org/elasticsearch/bootstrap/elasticsearch-version.help
+++ /dev/null
@@ -1,16 +0,0 @@
-NAME
-
-    version - Show version information and exit
-
-SYNOPSIS
-
-    elasticsearch version
-
-DESCRIPTION
-
-    This command shows Elasticsearch version, timestamp and build information as well as JVM info
-
-OPTIONS
-
-    -h,--help                    Shows this message
-
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/elasticsearch.help b/core/src/main/resources/org/elasticsearch/bootstrap/elasticsearch.help
deleted file mode 100644
index 83ee497dc21..00000000000
--- a/core/src/main/resources/org/elasticsearch/bootstrap/elasticsearch.help
+++ /dev/null
@@ -1,22 +0,0 @@
-NAME
-
-    elasticsearch - Manages elasticsearch
-
-SYNOPSIS
-
-    elasticsearch <command>
-
-DESCRIPTION
-
-    Start an elasticsearch node
-
-COMMANDS
-
-    start      Start elasticsearch
-
-    version    Show version information and exit
-
-NOTES
-
-    [*] For usage help on specific commands please type "elasticsearch <command> -h"
-
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
index 4909959015b..4845011ad05 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
@@ -31,7 +31,7 @@ grant codeBase "${codebase.securesm-1.0.jar}" {
 //// Very special jar permissions:
 //// These are dangerous permissions that we don't want to grant to everything.
 
-grant codeBase "${codebase.lucene-core-6.0.0-snapshot-bea235f.jar}" {
+grant codeBase "${codebase.lucene-core-6.0.0-snapshot-f0aa4fc.jar}" {
   // needed to allow MMapDirectory's "unmap hack" (die unmap hack, die)
   // java 8 package
   permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
@@ -72,9 +72,6 @@ grant {
   // set by ESTestCase to improve test reproducibility
   // TODO: set this with gradle or some other way that repros with seed?
   permission java.util.PropertyPermission "es.processors.override", "write";
-  // set by CLIToolTestCase
-  // TODO: do this differently? or test commandline tools differently?
-  permission java.util.PropertyPermission "es.default.path.home", "write";
 
   // TODO: these simply trigger a noisy warning if its unable to clear the properties
   // fix that in randomizedtesting
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
index fafa57118c2..8d56bc44b9a 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
@@ -31,7 +31,7 @@ grant codeBase "${codebase.securemock-1.2.jar}" {
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
 };
 
-grant codeBase "${codebase.lucene-test-framework-6.0.0-snapshot-bea235f.jar}" {
+grant codeBase "${codebase.lucene-test-framework-6.0.0-snapshot-f0aa4fc.jar}" {
   // needed by RamUsageTester
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
 };
diff --git a/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help b/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help
deleted file mode 100644
index ba39e1ab8fb..00000000000
--- a/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help
+++ /dev/null
@@ -1,59 +0,0 @@
-NAME
-
-    install - Install a plugin
-
-SYNOPSIS
-
-    plugin install <name or url>
-
-DESCRIPTION
-
-    This command installs an elasticsearch plugin. It can be used as follows:
-
-    Officially supported or commercial plugins require just the plugin name:
-
-        plugin install analysis-icu
-        plugin install x-pack
-
-    Plugins from Maven Central require 'groupId:artifactId:version':
-
-        plugin install org.elasticsearch:mapper-attachments:3.0.0
-
-    Plugins can be installed from a custom URL or file location as follows:
-
-        plugin install http://some.domain.name//my-plugin-1.0.0.zip
-        plugin install file:/path/to/my-plugin-1.0.0.zip
-
-OFFICIAL PLUGINS
-
-    The following plugins are officially supported and can be installed by just referring to their name
-
-    - analysis-icu
-    - analysis-kuromoji
-    - analysis-phonetic
-    - analysis-smartcn
-    - analysis-stempel
-    - delete-by-query
-    - discovery-azure
-    - discovery-ec2
-    - discovery-gce
-    - ingest-geoip
-    - lang-javascript
-    - lang-painless
-    - lang-python
-    - mapper-attachments
-    - mapper-murmur3
-    - mapper-size
-    - repository-azure
-    - repository-hdfs
-    - repository-s3
-    - store-smb
-
-
-OPTIONS
-
-    -v,--verbose                 Verbose output
-
-    -h,--help                    Shows this message
-
-    -b,--batch                   Enable batch mode explicitly, automatic confirmation of security permissions
diff --git a/core/src/main/resources/org/elasticsearch/plugins/plugin-list.help b/core/src/main/resources/org/elasticsearch/plugins/plugin-list.help
deleted file mode 100644
index c13949e8cb6..00000000000
--- a/core/src/main/resources/org/elasticsearch/plugins/plugin-list.help
+++ /dev/null
@@ -1,12 +0,0 @@
-NAME
-
-    list - List all plugins
-
-SYNOPSIS
-
-    plugin list
-
-DESCRIPTION
-
-    This command lists all installed elasticsearch plugins
-
diff --git a/core/src/main/resources/org/elasticsearch/plugins/plugin-remove.help b/core/src/main/resources/org/elasticsearch/plugins/plugin-remove.help
deleted file mode 100644
index b708adf1f69..00000000000
--- a/core/src/main/resources/org/elasticsearch/plugins/plugin-remove.help
+++ /dev/null
@@ -1,12 +0,0 @@
-NAME
-
-    remove - Remove a plugin
-
-SYNOPSIS
-
-    plugin remove <name>
-
-DESCRIPTION
-
-    This command removes an elasticsearch plugin
-
diff --git a/core/src/main/resources/org/elasticsearch/plugins/plugin.help b/core/src/main/resources/org/elasticsearch/plugins/plugin.help
deleted file mode 100644
index 5cba544627a..00000000000
--- a/core/src/main/resources/org/elasticsearch/plugins/plugin.help
+++ /dev/null
@@ -1,24 +0,0 @@
-NAME
-
-    plugin - Manages plugins
-
-SYNOPSIS
-
-    plugin <command>
-
-DESCRIPTION
-
-    Manage plugins
-
-COMMANDS
-
-    install    Install a plugin
-
-    remove     Remove a plugin
-
-    list       List installed plugins
-
-NOTES
-
-    [*] For usage help on specific commands please type "plugin <command> -h"
-
diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index 606ff09a60f..955eb309436 100644
--- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -63,7 +63,6 @@ import org.elasticsearch.indices.IndexTemplateAlreadyExistsException;
 import org.elasticsearch.indices.IndexTemplateMissingException;
 import org.elasticsearch.indices.InvalidIndexTemplateException;
 import org.elasticsearch.indices.recovery.RecoverFilesRecoveryException;
-import org.elasticsearch.percolator.PercolateException;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.admin.indices.alias.delete.AliasesNotFoundException;
@@ -382,19 +381,6 @@ public class ExceptionSerializationTests extends ESTestCase {
         assertEquals(id, ex.id());
     }
 
-    public void testPercolateException() throws IOException {
-        ShardId id = new ShardId("foo", "_na_", 1);
-        PercolateException ex = serialize(new PercolateException(id, "percolate my ass", null));
-        assertEquals(id, ex.getShardId());
-        assertEquals("percolate my ass", ex.getMessage());
-        assertNull(ex.getCause());
-
-        ex = serialize(new PercolateException(id, "percolate my ass", new NullPointerException()));
-        assertEquals(id, ex.getShardId());
-        assertEquals("percolate my ass", ex.getMessage());
-        assertTrue(ex.getCause() instanceof NullPointerException);
-    }
-
     public void testRoutingValidationException() throws IOException {
         RoutingTableValidation validation = new RoutingTableValidation();
         validation.addIndexFailure("foo", "bar");
@@ -746,7 +732,6 @@ public class ExceptionSerializationTests extends ESTestCase {
         ids.put(85, org.elasticsearch.index.AlreadyExpiredException.class);
         ids.put(86, org.elasticsearch.search.aggregations.AggregationExecutionException.class);
         ids.put(88, org.elasticsearch.indices.InvalidIndexTemplateException.class);
-        ids.put(89, org.elasticsearch.percolator.PercolateException.class);
         ids.put(90, org.elasticsearch.index.engine.RefreshFailedEngineException.class);
         ids.put(91, org.elasticsearch.search.aggregations.AggregationInitializationException.class);
         ids.put(92, org.elasticsearch.indices.recovery.DelayRecoveryException.class);
diff --git a/core/src/test/java/org/elasticsearch/VersionTests.java b/core/src/test/java/org/elasticsearch/VersionTests.java
index 7824ecd39b1..65c91f5daab 100644
--- a/core/src/test/java/org/elasticsearch/VersionTests.java
+++ b/core/src/test/java/org/elasticsearch/VersionTests.java
@@ -32,7 +32,7 @@ import java.util.Locale;
 import java.util.Map;
 
 import static org.elasticsearch.Version.V_2_2_0;
-import static org.elasticsearch.Version.V_5_0_0;
+import static org.elasticsearch.Version.V_5_0_0_alpha1;
 import static org.elasticsearch.test.VersionUtils.randomVersion;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.containsString;
@@ -42,21 +42,21 @@ import static org.hamcrest.Matchers.sameInstance;
 public class VersionTests extends ESTestCase {
 
     public void testVersionComparison() throws Exception {
-        assertThat(V_2_2_0.before(V_5_0_0), is(true));
+        assertThat(V_2_2_0.before(V_5_0_0_alpha1), is(true));
         assertThat(V_2_2_0.before(V_2_2_0), is(false));
-        assertThat(V_5_0_0.before(V_2_2_0), is(false));
+        assertThat(V_5_0_0_alpha1.before(V_2_2_0), is(false));
 
-        assertThat(V_2_2_0.onOrBefore(V_5_0_0), is(true));
+        assertThat(V_2_2_0.onOrBefore(V_5_0_0_alpha1), is(true));
         assertThat(V_2_2_0.onOrBefore(V_2_2_0), is(true));
-        assertThat(V_5_0_0.onOrBefore(V_2_2_0), is(false));
+        assertThat(V_5_0_0_alpha1.onOrBefore(V_2_2_0), is(false));
 
-        assertThat(V_2_2_0.after(V_5_0_0), is(false));
+        assertThat(V_2_2_0.after(V_5_0_0_alpha1), is(false));
         assertThat(V_2_2_0.after(V_2_2_0), is(false));
-        assertThat(V_5_0_0.after(V_2_2_0), is(true));
+        assertThat(V_5_0_0_alpha1.after(V_2_2_0), is(true));
 
-        assertThat(V_2_2_0.onOrAfter(V_5_0_0), is(false));
+        assertThat(V_2_2_0.onOrAfter(V_5_0_0_alpha1), is(false));
         assertThat(V_2_2_0.onOrAfter(V_2_2_0), is(true));
-        assertThat(V_5_0_0.onOrAfter(V_2_2_0), is(true));
+        assertThat(V_5_0_0_alpha1.onOrAfter(V_2_2_0), is(true));
 
         assertTrue(Version.fromString("5.0.0-alpha2").onOrAfter(Version.fromString("5.0.0-alpha1")));
         assertTrue(Version.fromString("5.0.0").onOrAfter(Version.fromString("5.0.0-beta2")));
@@ -133,7 +133,7 @@ public class VersionTests extends ESTestCase {
 
     public void testIndexCreatedVersion() {
         // an actual index has a IndexMetaData.SETTING_INDEX_UUID
-        final Version version = randomFrom(Version.V_2_0_0, Version.V_2_3_0, Version.V_5_0_0);
+        final Version version = randomFrom(Version.V_2_0_0, Version.V_2_3_0, Version.V_5_0_0_alpha1);
         assertEquals(version, Version.indexCreated(Settings.builder().put(IndexMetaData.SETTING_INDEX_UUID, "foo").put(IndexMetaData.SETTING_VERSION_CREATED, version).build()));
     }
 
@@ -142,13 +142,13 @@ public class VersionTests extends ESTestCase {
         assertThat(Version.V_2_1_0.minimumCompatibilityVersion(), equalTo(Version.V_2_0_0));
         assertThat(Version.V_2_2_0.minimumCompatibilityVersion(), equalTo(Version.V_2_0_0));
         assertThat(Version.V_2_3_0.minimumCompatibilityVersion(), equalTo(Version.V_2_0_0));
-        assertThat(Version.V_5_0_0.minimumCompatibilityVersion(), equalTo(Version.V_5_0_0));
+        assertThat(Version.V_5_0_0_alpha1.minimumCompatibilityVersion(), equalTo(Version.V_5_0_0_alpha1));
     }
 
     public void testToString() {
         // with 2.0.beta we lowercase
         assertEquals("2.0.0-beta1", Version.V_2_0_0_beta1.toString());
-        assertEquals("5.0.0", Version.V_5_0_0.toString());
+        assertEquals("5.0.0-alpha1", Version.V_5_0_0_alpha1.toString());
         assertEquals("2.3.0", Version.V_2_3_0.toString());
         assertEquals("0.90.0.Beta1", Version.fromString("0.90.0.Beta1").toString());
         assertEquals("1.0.0.Beta1", Version.fromString("1.0.0.Beta1").toString());
@@ -231,7 +231,7 @@ public class VersionTests extends ESTestCase {
                 assertTrue(constantName + " should be final", Modifier.isFinal(versionConstant.getModifiers()));
 
                 Version v = (Version) versionConstant.get(Version.class);
-                logger.info("Checking " + v);
+                logger.info("Checking {}", v);
                 assertEquals("Version id " + field.getName() + " does not point to " + constantName, v, Version.fromId(versionId));
                 assertEquals("Version " + constantName + " does not have correct id", versionId, v.id);
                 if (v.major >= 2) {
@@ -240,6 +240,8 @@ public class VersionTests extends ESTestCase {
                         number = number.replace("-beta", "_beta");
                     } else if (v.isRC()) {
                         number = number.replace("-rc", "_rc");
+                    } else if (v.isAlpha()) {
+                        number = number.replace("-alpha", "_alpha");
                     }
                     assertEquals("V_" + number.replace('.', '_'), constantName);
                 } else {
@@ -260,4 +262,20 @@ public class VersionTests extends ESTestCase {
         }
     }
 
+    // this test ensures we never bump the lucene version in a bugfix release
+    public void testLuceneVersionIsSameOnMinorRelease() {
+        for (Version version : VersionUtils.allVersions()) {
+            for (Version other : VersionUtils.allVersions()) {
+                if (other.onOrAfter(version)) {
+                    assertTrue("lucene versions must be "  + other + " >= " + version,
+                        other.luceneVersion.onOrAfter(version.luceneVersion));
+                }
+                if (other.major == version.major && other.minor == version.minor) {
+                    assertEquals(other.luceneVersion.major, version.luceneVersion.major);
+                    assertEquals(other.luceneVersion.minor, version.luceneVersion.minor);
+                    // should we also assert the lucene bugfix version?
+                }
+            }
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
index 5109ab979cf..379cb5942a2 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
@@ -29,8 +29,8 @@ import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -49,6 +49,7 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicReference;
 
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
@@ -237,8 +238,8 @@ public class CancellableTasksTests extends TaskManagerTestCase {
 
         // Cancel main task
         CancelTasksRequest request = new CancelTasksRequest();
-        request.reason("Testing Cancellation");
-        request.taskId(new TaskId(testNodes[0].discoveryNode.getId(), mainTask.getId()));
+        request.setReason("Testing Cancellation");
+        request.setTaskId(new TaskId(testNodes[0].discoveryNode.getId(), mainTask.getId()));
         // And send the cancellation request to a random node
         CancelTasksResponse response = testNodes[randomIntBetween(0, testNodes.length - 1)].transportCancelTasksAction.execute(request)
             .get();
@@ -270,7 +271,7 @@ public class CancellableTasksTests extends TaskManagerTestCase {
 
         // Make sure that tasks are no longer running
         ListTasksResponse listTasksResponse = testNodes[randomIntBetween(0, testNodes.length - 1)]
-            .transportListTasksAction.execute(new ListTasksRequest().taskId(
+            .transportListTasksAction.execute(new ListTasksRequest().setTaskId(
                 new TaskId(testNodes[0].discoveryNode.getId(), mainTask.getId()))).get();
         assertEquals(0, listTasksResponse.getTasks().size());
 
@@ -313,7 +314,7 @@ public class CancellableTasksTests extends TaskManagerTestCase {
 
         // Make sure that tasks are running
         ListTasksResponse listTasksResponse = testNodes[randomIntBetween(0, testNodes.length - 1)]
-            .transportListTasksAction.execute(new ListTasksRequest().parentTaskId(new TaskId(mainNode, mainTask.getId()))).get();
+            .transportListTasksAction.execute(new ListTasksRequest().setParentTaskId(new TaskId(mainNode, mainTask.getId()))).get();
         assertThat(listTasksResponse.getTasks().size(), greaterThanOrEqualTo(blockOnNodes.size()));
 
         // Simulate the coordinating node leaving the cluster
@@ -324,15 +325,15 @@ public class CancellableTasksTests extends TaskManagerTestCase {
         DiscoveryNode master = discoveryNodes[0];
         for (int i = 1; i < testNodes.length; i++) {
             // Notify only nodes that should remain in the cluster
-            testNodes[i].clusterService.setState(ClusterStateCreationUtils.state(testNodes[i].discoveryNode, master, discoveryNodes));
+            setState(testNodes[i].clusterService, ClusterStateCreationUtils.state(testNodes[i].discoveryNode, master, discoveryNodes));
         }
 
         if (simulateBanBeforeLeaving) {
             logger.info("--> Simulate issuing cancel request on the node that is about to leave the cluster");
             // Simulate issuing cancel request on the node that is about to leave the cluster
             CancelTasksRequest request = new CancelTasksRequest();
-            request.reason("Testing Cancellation");
-            request.taskId(new TaskId(testNodes[0].discoveryNode.getId(), mainTask.getId()));
+            request.setReason("Testing Cancellation");
+            request.setTaskId(new TaskId(testNodes[0].discoveryNode.getId(), mainTask.getId()));
             // And send the cancellation request to a random node
             CancelTasksResponse response = testNodes[0].transportCancelTasksAction.execute(request).get();
             logger.info("--> Done simulating issuing cancel request on the node that is about to leave the cluster");
@@ -356,7 +357,7 @@ public class CancellableTasksTests extends TaskManagerTestCase {
             // Make sure that tasks are no longer running
             try {
                 ListTasksResponse listTasksResponse1 = testNodes[randomIntBetween(1, testNodes.length - 1)]
-                    .transportListTasksAction.execute(new ListTasksRequest().taskId(new TaskId(mainNode, mainTask.getId()))).get();
+                    .transportListTasksAction.execute(new ListTasksRequest().setTaskId(new TaskId(mainNode, mainTask.getId()))).get();
                 assertEquals(0, listTasksResponse1.getTasks().size());
             } catch (InterruptedException ex) {
                 Thread.currentThread().interrupt();
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
index 4dcf54b5d0b..3d996becbae 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
@@ -30,9 +30,9 @@ import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -40,7 +40,6 @@ import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.cluster.TestClusterService;
 import org.elasticsearch.test.tasks.MockTaskManager;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -58,6 +57,9 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+
 /**
  * The test case for unit testing task manager and related transport actions
  */
@@ -137,14 +139,14 @@ public abstract class TaskManagerTestCase extends ESTestCase {
      * Simulates node-based task that can be used to block node tasks so they are guaranteed to be registered by task manager
      */
     abstract class AbstractTestNodesAction<NodesRequest extends BaseNodesRequest<NodesRequest>, NodeRequest extends BaseNodeRequest>
-        extends TransportNodesAction<NodesRequest, NodesResponse, NodeRequest, NodeResponse> {
+            extends TransportNodesAction<NodesRequest, NodesResponse, NodeRequest, NodeResponse> {
 
         AbstractTestNodesAction(Settings settings, String actionName, ClusterName clusterName, ThreadPool threadPool,
                                 ClusterService clusterService, TransportService transportService, Supplier<NodesRequest> request,
                                 Supplier<NodeRequest> nodeRequest) {
             super(settings, actionName, clusterName, threadPool, clusterService, transportService,
-                new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(Settings.EMPTY),
-                request, nodeRequest, ThreadPool.Names.GENERIC);
+                    new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(Settings.EMPTY),
+                    request, nodeRequest, ThreadPool.Names.GENERIC);
         }
 
         @Override
@@ -182,8 +184,8 @@ public abstract class TaskManagerTestCase extends ESTestCase {
     public static class TestNode implements Releasable {
         public TestNode(String name, ThreadPool threadPool, Settings settings) {
             transportService = new TransportService(settings,
-                new LocalTransport(settings, threadPool, Version.CURRENT, new NamedWriteableRegistry()),
-                threadPool, new NamedWriteableRegistry()) {
+                    new LocalTransport(settings, threadPool, Version.CURRENT, new NamedWriteableRegistry()),
+                    threadPool) {
                 @Override
                 protected TaskManager createTaskManager() {
                     if (MockTaskManager.USE_MOCK_TASK_MANAGER_SETTING.get(settings)) {
@@ -194,19 +196,19 @@ public abstract class TaskManagerTestCase extends ESTestCase {
                 }
             };
             transportService.start();
-            clusterService = new TestClusterService(threadPool, transportService);
+            clusterService = createClusterService(threadPool);
             clusterService.add(transportService.getTaskManager());
             discoveryNode = new DiscoveryNode(name, transportService.boundAddress().publishAddress(), Version.CURRENT);
             IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(settings);
             ActionFilters actionFilters = new ActionFilters(Collections.emptySet());
             transportListTasksAction = new TransportListTasksAction(settings, clusterName, threadPool, clusterService, transportService,
-                actionFilters, indexNameExpressionResolver);
+                    actionFilters, indexNameExpressionResolver);
             transportCancelTasksAction = new TransportCancelTasksAction(settings, clusterName, threadPool, clusterService, transportService,
-                actionFilters, indexNameExpressionResolver);
+                    actionFilters, indexNameExpressionResolver);
             transportService.acceptIncomingRequests();
         }
 
-        public final TestClusterService clusterService;
+        public final ClusterService clusterService;
         public final TransportService transportService;
         public final DiscoveryNode discoveryNode;
         public final TransportListTasksAction transportListTasksAction;
@@ -214,8 +216,13 @@ public abstract class TaskManagerTestCase extends ESTestCase {
 
         @Override
         public void close() {
+            clusterService.close();
             transportService.close();
         }
+
+        public String getNodeId() {
+            return discoveryNode.getId();
+        }
     }
 
     public static void connectNodes(TestNode... nodes) {
@@ -225,7 +232,7 @@ public abstract class TaskManagerTestCase extends ESTestCase {
         }
         DiscoveryNode master = discoveryNodes[0];
         for (TestNode node : nodes) {
-            node.clusterService.setState(ClusterStateCreationUtils.state(node.discoveryNode, master, discoveryNodes));
+            setState(node.clusterService, ClusterStateCreationUtils.state(node.discoveryNode, master, discoveryNodes));
         }
         for (TestNode nodeA : nodes) {
             for (TestNode nodeB : nodes) {
@@ -238,7 +245,7 @@ public abstract class TaskManagerTestCase extends ESTestCase {
         RecordingTaskManagerListener[] listeners = new RecordingTaskManagerListener[nodes.length];
         for (int i = 0; i < nodes.length; i++) {
             listeners[i] = new RecordingTaskManagerListener(nodes[i].discoveryNode, actionMasks);
-            ((MockTaskManager) (nodes[i].clusterService.getTaskManager())).addListener(listeners[i]);
+            ((MockTaskManager) (nodes[i].transportService.getTaskManager())).addListener(listeners[i]);
         }
         return listeners;
     }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
index eaa3caf9084..e9ade3ef5af 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
@@ -18,7 +18,10 @@
  */
 package org.elasticsearch.action.admin.cluster.node.tasks;
 
+import org.elasticsearch.ElasticsearchTimeoutException;
+import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.ListenableActionFuture;
+import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction;
@@ -27,10 +30,10 @@ import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
 import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
 import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeAction;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction;
+import org.elasticsearch.action.fieldstats.FieldStatsAction;
 import org.elasticsearch.action.index.IndexAction;
-import org.elasticsearch.action.percolate.PercolateAction;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
@@ -40,6 +43,8 @@ import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.tasks.MockTaskManager;
 import org.elasticsearch.test.tasks.MockTaskManagerListener;
 import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.transport.ReceiveTimeoutTransportException;
+import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -54,8 +59,12 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Function;
 
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
+import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.emptyCollectionOf;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 
@@ -111,27 +120,27 @@ public class TasksIT extends ESIntegTestCase {
 
         // Verify that one of these tasks is a parent of another task
         if (tasks.get(0).getParentTaskId().isSet()) {
-            assertParentTask(Collections.singletonList(tasks.get(1)), tasks.get(0));
-        } else {
             assertParentTask(Collections.singletonList(tasks.get(0)), tasks.get(1));
+        } else {
+            assertParentTask(Collections.singletonList(tasks.get(1)), tasks.get(0));
         }
     }
 
     public void testTransportReplicationAllShardsTasks() {
-        registerTaskManageListeners(PercolateAction.NAME);  // main task
-        registerTaskManageListeners(PercolateAction.NAME + "[s]"); // shard level tasks
+        registerTaskManageListeners(FieldStatsAction.NAME);  // main task
+        registerTaskManageListeners(FieldStatsAction.NAME + "[s]"); // shard level tasks
         createIndex("test");
         ensureGreen("test"); // Make sure all shards are allocated
-        client().preparePercolate().setIndices("test").setDocumentType("foo").setSource("{}").get();
+        client().prepareFieldStats().setFields("field").get();
 
         // the percolate operation should produce one main task
         NumShards numberOfShards = getNumShards("test");
-        assertEquals(1, numberOfEvents(PercolateAction.NAME, Tuple::v1));
+        assertEquals(1, numberOfEvents(FieldStatsAction.NAME, Tuple::v1));
         // and then one operation per shard
-        assertEquals(numberOfShards.totalNumShards, numberOfEvents(PercolateAction.NAME + "[s]", Tuple::v1));
+        assertEquals(numberOfShards.totalNumShards, numberOfEvents(FieldStatsAction.NAME + "[s]", Tuple::v1));
 
         // the shard level tasks should have the main task as a parent
-        assertParentTask(findEvents(PercolateAction.NAME + "[s]", Tuple::v1), findEvents(PercolateAction.NAME, Tuple::v1).get(0));
+        assertParentTask(findEvents(FieldStatsAction.NAME + "[s]", Tuple::v1), findEvents(FieldStatsAction.NAME, Tuple::v1).get(0));
     }
 
     public void testTransportBroadcastByNodeTasks() {
@@ -257,8 +266,8 @@ public class TasksIT extends ESIntegTestCase {
         ReentrantLock taskFinishLock = new ReentrantLock();
         taskFinishLock.lock();
         CountDownLatch taskRegistered = new CountDownLatch(1);
-        for (ClusterService clusterService : internalCluster().getInstances(ClusterService.class)) {
-            ((MockTaskManager)clusterService.getTaskManager()).addListener(new MockTaskManagerListener() {
+        for (TransportService transportService : internalCluster().getInstances(TransportService.class)) {
+            ((MockTaskManager) transportService.getTaskManager()).addListener(new MockTaskManagerListener() {
                 @Override
                 public void onTaskRegistered(Task task) {
                     if (task.getAction().startsWith(IndexAction.NAME)) {
@@ -327,10 +336,93 @@ public class TasksIT extends ESIntegTestCase {
         assertEquals(0, client().admin().cluster().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size());
     }
 
+    public void testTasksListWaitForCompletion() throws Exception {
+        // Start blocking test task
+        ListenableActionFuture<TestTaskPlugin.NodesResponse> future = TestTaskPlugin.TestTaskAction.INSTANCE.newRequestBuilder(client())
+                .execute();
+
+        ListenableActionFuture<ListTasksResponse> waitResponseFuture;
+        try {
+            // Wait for the task to start on all nodes
+            assertBusy(() -> assertEquals(internalCluster().numDataAndMasterNodes(),
+                client().admin().cluster().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size()));
+
+            // Spin up a request to wait for that task to finish
+            waitResponseFuture = client().admin().cluster().prepareListTasks()
+                    .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").setWaitForCompletion(true).execute();
+        } finally {
+            // Unblock the request so the wait for completion request can finish
+            TestTaskPlugin.UnblockTestTasksAction.INSTANCE.newRequestBuilder(client()).get();
+        }
+
+        // Now that the task is unblocked the list response will come back
+        ListTasksResponse waitResponse = waitResponseFuture.get();
+        // If any tasks come back then they are the tasks we asked for - it'd be super weird if this wasn't true
+        for (TaskInfo task: waitResponse.getTasks()) {
+            assertEquals(task.getAction(), TestTaskPlugin.TestTaskAction.NAME + "[n]");
+        }
+        // See the next test to cover the timeout case
+
+        future.get();
+    }
+
+    public void testTasksListWaitForTimeout() throws Exception {
+        // Start blocking test task
+        ListenableActionFuture<TestTaskPlugin.NodesResponse> future = TestTaskPlugin.TestTaskAction.INSTANCE.newRequestBuilder(client())
+                .execute();
+        try {
+            // Wait for the task to start on all nodes
+            assertBusy(() -> assertEquals(internalCluster().numDataAndMasterNodes(),
+                client().admin().cluster().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size()));
+
+            // Spin up a request that should wait for those tasks to finish
+            // It will timeout because we haven't unblocked the tasks
+            ListTasksResponse waitResponse = client().admin().cluster().prepareListTasks()
+                    .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").setWaitForCompletion(true).setTimeout(timeValueMillis(100))
+                    .get();
+
+            assertFalse(waitResponse.getNodeFailures().isEmpty());
+            for (FailedNodeException failure : waitResponse.getNodeFailures()) {
+                Throwable timeoutException = failure.getCause();
+                // The exception sometimes comes back wrapped depending on the client
+                if (timeoutException.getCause() != null) {
+                    timeoutException = timeoutException.getCause();
+                }
+                assertThat(timeoutException,
+                        either(instanceOf(ElasticsearchTimeoutException.class)).or(instanceOf(ReceiveTimeoutTransportException.class)));
+            }
+        } finally {
+            // Now we can unblock those requests
+            TestTaskPlugin.UnblockTestTasksAction.INSTANCE.newRequestBuilder(client()).get();
+        }
+        future.get();
+    }
+
+    public void testTasksListWaitForNoTask() throws Exception {
+        // Spin up a request to wait for no matching tasks
+        ListenableActionFuture<ListTasksResponse> waitResponseFuture = client().admin().cluster().prepareListTasks()
+                .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").setWaitForCompletion(true).setTimeout(timeValueMillis(10))
+                .execute();
+
+        // It should finish quickly and without complaint
+        assertThat(waitResponseFuture.get().getTasks(), emptyCollectionOf(TaskInfo.class));
+    }
+
+    public void testTasksWaitForAllTask() throws Exception {
+        // Spin up a request to wait for all tasks in the cluster to make sure it doesn't cause an infinite loop
+        ListTasksResponse response = client().admin().cluster().prepareListTasks().setWaitForCompletion(true)
+            .setTimeout(timeValueSeconds(10)).get();
+
+        // It should finish quickly and without complaint and list the list tasks themselves
+        assertThat(response.getNodeFailures(), emptyCollectionOf(FailedNodeException.class));
+        assertThat(response.getTaskFailures(), emptyCollectionOf(TaskOperationFailure.class));
+        assertThat(response.getTasks().size(), greaterThanOrEqualTo(1));
+    }
+
     @Override
     public void tearDown() throws Exception {
         for (Map.Entry<Tuple<String, String>, RecordingTaskManagerListener> entry : listeners.entrySet()) {
-            ((MockTaskManager)internalCluster().getInstance(ClusterService.class, entry.getKey().v1()).getTaskManager()).removeListener(entry.getValue());
+            ((MockTaskManager) internalCluster().getInstance(TransportService.class, entry.getKey().v1()).getTaskManager()).removeListener(entry.getValue());
         }
         listeners.clear();
         super.tearDown();
@@ -340,10 +432,10 @@ public class TasksIT extends ESIntegTestCase {
      * Registers recording task event listeners with the given action mask on all nodes
      */
     private void registerTaskManageListeners(String actionMasks) {
-        for (ClusterService clusterService : internalCluster().getInstances(ClusterService.class)) {
-            DiscoveryNode node = clusterService.localNode();
+        for (String nodeName : internalCluster().getNodeNames()) {
+            DiscoveryNode node = internalCluster().getInstance(ClusterService.class, nodeName).localNode();
             RecordingTaskManagerListener listener = new RecordingTaskManagerListener(node, Strings.splitStringToArray(actionMasks, ','));
-            ((MockTaskManager)clusterService.getTaskManager()).addListener(listener);
+            ((MockTaskManager) internalCluster().getInstance(TransportService.class, nodeName).getTaskManager()).addListener(listener);
             RecordingTaskManagerListener oldListener = listeners.put(new Tuple<>(node.name(), actionMasks), listener);
             assertNull(oldListener);
         }
@@ -395,7 +487,7 @@ public class TasksIT extends ESIntegTestCase {
      */
     private void assertParentTask(List<TaskInfo> tasks, TaskInfo parentTask) {
         for (TaskInfo task : tasks) {
-            assertFalse(task.getParentTaskId().isSet());
+            assertTrue(task.getParentTaskId().isSet());
             assertEquals(parentTask.getNode().getId(), task.getParentTaskId().getNodeId());
             assertTrue(Strings.hasLength(task.getParentTaskId().getNodeId()));
             assertEquals(parentTask.getId(), task.getParentTaskId().getId());
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
index 0d4372a51eb..72ea730f881 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
@@ -35,10 +35,10 @@ import org.elasticsearch.action.support.tasks.BaseTasksResponse;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -345,7 +345,10 @@ public class TestTaskPlugin extends Plugin {
 
 
     public static class UnblockTestTasksRequest extends BaseTasksRequest<UnblockTestTasksRequest> {
-
+        @Override
+        public boolean match(Task task) {
+            return task instanceof TestTask && super.match(task);
+        }
     }
 
     public static class UnblockTestTasksResponse extends BaseTasksResponse {
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
index e1501f9b14c..972d9735efb 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
@@ -35,9 +35,10 @@ import org.elasticsearch.action.support.tasks.BaseTasksRequest;
 import org.elasticsearch.action.support.tasks.BaseTasksResponse;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -48,13 +49,13 @@ import org.elasticsearch.test.tasks.MockTaskManager;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
@@ -294,14 +295,14 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
             actions[i] = new TestNodesAction(Settings.EMPTY, "testAction", clusterName, threadPool, testNodes[i].clusterService, testNodes[i].transportService) {
                 @Override
                 protected NodeResponse nodeOperation(NodeRequest request) {
-                    logger.info("Action on node " + node);
+                    logger.info("Action on node {}", node);
                     actionLatch.countDown();
                     try {
                         checkLatch.await();
                     } catch (InterruptedException ex) {
                         Thread.currentThread().interrupt();
                     }
-                    logger.info("Action on node " + node + " finished");
+                    logger.info("Action on node {} finished", node);
                     return new NodeResponse(testNodes[node].discoveryNode);
                 }
             };
@@ -355,7 +356,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         int testNodeNum = randomIntBetween(0, testNodes.length - 1);
         TestNode testNode = testNodes[testNodeNum];
         ListTasksRequest listTasksRequest = new ListTasksRequest();
-        listTasksRequest.actions("testAction*"); // pick all test actions
+        listTasksRequest.setActions("testAction*"); // pick all test actions
         logger.info("Listing currently running tasks using node [{}]", testNodeNum);
         ListTasksResponse response = testNode.transportListTasksAction.execute(listTasksRequest).get();
         logger.info("Checking currently running tasks");
@@ -371,7 +372,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         // Check task counts using transport with filtering
         testNode = testNodes[randomIntBetween(0, testNodes.length - 1)];
         listTasksRequest = new ListTasksRequest();
-        listTasksRequest.actions("testAction[n]"); // only pick node actions
+        listTasksRequest.setActions("testAction[n]"); // only pick node actions
         response = testNode.transportListTasksAction.execute(listTasksRequest).get();
         assertEquals(testNodes.length, response.getPerNodeTasks().size());
         for (Map.Entry<DiscoveryNode, List<TaskInfo>> entry : response.getPerNodeTasks().entrySet()) {
@@ -380,7 +381,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         }
 
         // Check task counts using transport with detailed description
-        listTasksRequest.detailed(true); // same request only with detailed description
+        listTasksRequest.setDetailed(true); // same request only with detailed description
         response = testNode.transportListTasksAction.execute(listTasksRequest).get();
         assertEquals(testNodes.length, response.getPerNodeTasks().size());
         for (Map.Entry<DiscoveryNode, List<TaskInfo>> entry : response.getPerNodeTasks().entrySet()) {
@@ -389,7 +390,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         }
 
         // Make sure that the main task on coordinating node is the task that was returned to us by execute()
-        listTasksRequest.actions("testAction"); // only pick the main task
+        listTasksRequest.setActions("testAction"); // only pick the main task
         response = testNode.transportListTasksAction.execute(listTasksRequest).get();
         assertEquals(1, response.getTasks().size());
         assertEquals(mainTask.getId(), response.getTasks().get(0).getId());
@@ -417,7 +418,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
 
         // Get the parent task
         ListTasksRequest listTasksRequest = new ListTasksRequest();
-        listTasksRequest.actions("testAction");
+        listTasksRequest.setActions("testAction");
         ListTasksResponse response = testNode.transportListTasksAction.execute(listTasksRequest).get();
         assertEquals(1, response.getTasks().size());
         String parentNode = response.getTasks().get(0).getNode().getId();
@@ -425,7 +426,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
 
         // Find tasks with common parent
         listTasksRequest = new ListTasksRequest();
-        listTasksRequest.parentTaskId(new TaskId(parentNode, parentTaskId));
+        listTasksRequest.setParentTaskId(new TaskId(parentNode, parentTaskId));
         response = testNode.transportListTasksAction.execute(listTasksRequest).get();
         assertEquals(testNodes.length, response.getTasks().size());
         for (TaskInfo task : response.getTasks()) {
@@ -451,7 +452,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
 
         // Get the parent task
         ListTasksRequest listTasksRequest = new ListTasksRequest();
-        listTasksRequest.actions("testAction*");
+        listTasksRequest.setActions("testAction*");
         ListTasksResponse response = testNode.transportListTasksAction.execute(listTasksRequest).get();
         assertEquals(0, response.getTasks().size());
 
@@ -472,7 +473,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         // Check task counts using transport with filtering
         TestNode testNode = testNodes[randomIntBetween(0, testNodes.length - 1)];
         ListTasksRequest listTasksRequest = new ListTasksRequest();
-        listTasksRequest.actions("testAction[n]"); // only pick node actions
+        listTasksRequest.setActions("testAction[n]"); // only pick node actions
         ListTasksResponse response = testNode.transportListTasksAction.execute(listTasksRequest).get();
         assertEquals(testNodes.length, response.getPerNodeTasks().size());
         for (Map.Entry<DiscoveryNode, List<TaskInfo>> entry : response.getPerNodeTasks().entrySet()) {
@@ -482,7 +483,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
 
         // Check task counts using transport with detailed description
         long minimalDurationNanos = System.nanoTime() - maximumStartTimeNanos;
-        listTasksRequest.detailed(true); // same request only with detailed description
+        listTasksRequest.setDetailed(true); // same request only with detailed description
         response = testNode.transportListTasksAction.execute(listTasksRequest).get();
         assertEquals(testNodes.length, response.getPerNodeTasks().size());
         for (Map.Entry<DiscoveryNode, List<TaskInfo>> entry : response.getPerNodeTasks().entrySet()) {
@@ -518,9 +519,9 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
 
         // Try to cancel main task using action name
         CancelTasksRequest request = new CancelTasksRequest();
-        request.nodesIds(testNodes[0].discoveryNode.getId());
-        request.reason("Testing Cancellation");
-        request.actions(actionName);
+        request.setNodesIds(testNodes[0].discoveryNode.getId());
+        request.setReason("Testing Cancellation");
+        request.setActions(actionName);
         CancelTasksResponse response = testNodes[randomIntBetween(0, testNodes.length - 1)].transportCancelTasksAction.execute(request)
             .get();
 
@@ -532,8 +533,8 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
 
         // Try to cancel main task using id
         request = new CancelTasksRequest();
-        request.reason("Testing Cancellation");
-        request.taskId(new TaskId(testNodes[0].discoveryNode.getId(), task.getId()));
+        request.setReason("Testing Cancellation");
+        request.setTaskId(new TaskId(testNodes[0].discoveryNode.getId(), task.getId()));
         response = testNodes[randomIntBetween(0, testNodes.length - 1)].transportCancelTasksAction.execute(request).get();
 
         // Shouldn't match any tasks since testAction doesn't support cancellation
@@ -544,7 +545,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
 
         // Make sure that task is still running
         ListTasksRequest listTasksRequest = new ListTasksRequest();
-        listTasksRequest.actions(actionName);
+        listTasksRequest.setActions(actionName);
         ListTasksResponse listResponse = testNodes[randomIntBetween(0, testNodes.length - 1)].transportListTasksAction.execute
             (listTasksRequest).get();
         assertEquals(1, listResponse.getPerNodeTasks().size());
@@ -565,7 +566,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
             actions[i] = new TestNodesAction(Settings.EMPTY, "testAction", clusterName, threadPool, testNodes[i].clusterService, testNodes[i].transportService) {
                 @Override
                 protected NodeResponse nodeOperation(NodeRequest request) {
-                    logger.info("Action on node " + node);
+                    logger.info("Action on node {}", node);
                     throw new RuntimeException("Test exception");
                 }
             };
@@ -604,9 +605,9 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
             tasksActions[i] = new TestTasksAction(Settings.EMPTY, "testTasksAction", clusterName, threadPool, testNodes[i].clusterService, testNodes[i].transportService) {
                 @Override
                 protected TestTaskResponse taskOperation(TestTasksRequest request, Task task) {
-                    logger.info("Task action on node " + node);
-                    if (failTaskOnNode == node && task.getParentTaskId().isSet() == false) {
-                        logger.info("Failing on node " + node);
+                    logger.info("Task action on node {}", node);
+                    if (failTaskOnNode == node && task.getParentTaskId().isSet()) {
+                        logger.info("Failing on node {}", node);
                         throw new RuntimeException("Task level failure");
                     }
                     return new TestTaskResponse("Success on node " + node);
@@ -617,7 +618,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         // Run task action on node tasks that are currently running
         // should be successful on all nodes except one
         TestTasksRequest testTasksRequest = new TestTasksRequest();
-        testTasksRequest.actions("testAction[n]"); // pick all test actions
+        testTasksRequest.setActions("testAction[n]"); // pick all test actions
         TestTasksResponse response = tasksActions[0].execute(testTasksRequest).get();
         // Get successful responses from all nodes except one
         assertEquals(testNodes.length - 1, response.tasks.size());
@@ -630,4 +631,76 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         NodesResponse responses = future.get();
         assertEquals(0, responses.failureCount());
     }
+
+
+    /**
+     * This test starts nodes actions that blocks on all nodes. While node actions are blocked in the middle of execution
+     * it executes a tasks action that targets these blocked node actions. The test verifies that task actions are only
+     * getting executed on nodes that are not listed in the node filter.
+     */
+    public void testTaskNodeFiltering() throws ExecutionException, InterruptedException, IOException {
+        setupTestNodes(Settings.EMPTY);
+        connectNodes(testNodes);
+        CountDownLatch checkLatch = new CountDownLatch(1);
+        // Start some test nodes action so we could have something to run tasks actions on
+        ActionFuture<NodesResponse> future = startBlockingTestNodesAction(checkLatch);
+
+        String[] allNodes = new String[testNodes.length];
+        for (int i = 0; i < testNodes.length; i++) {
+            allNodes[i] = testNodes[i].getNodeId();
+        }
+
+        int filterNodesSize = randomInt(allNodes.length);
+        Set<String> filterNodes = new HashSet<>(randomSubsetOf(filterNodesSize, allNodes));
+        logger.info("Filtering out nodes {} size: {}", filterNodes, filterNodesSize);
+
+        TestTasksAction[] tasksActions = new TestTasksAction[nodesCount];
+        for (int i = 0; i < testNodes.length; i++) {
+            final int node = i;
+            // Simulate a task action that works on all nodes except nodes listed in filterNodes.
+            // We are testing that it works.
+            tasksActions[i] = new TestTasksAction(Settings.EMPTY, "testTasksAction", clusterName, threadPool,
+                testNodes[i].clusterService, testNodes[i].transportService) {
+
+                @Override
+                protected String[] filterNodeIds(DiscoveryNodes nodes, String[] nodesIds) {
+                    String[] superNodes = super.filterNodeIds(nodes, nodesIds);
+                    List<String> filteredNodes = new ArrayList<>();
+                    for (String node : superNodes) {
+                        if (filterNodes.contains(node) == false) {
+                            filteredNodes.add(node);
+                        }
+                    }
+                    return filteredNodes.toArray(new String[filteredNodes.size()]);
+                }
+
+                @Override
+                protected TestTaskResponse taskOperation(TestTasksRequest request, Task task) {
+                    return new TestTaskResponse(testNodes[node].getNodeId());
+                }
+            };
+        }
+
+        // Run task action on node tasks that are currently running
+        // should be successful on all nodes except nodes that we filtered out
+        TestTasksRequest testTasksRequest = new TestTasksRequest();
+        testTasksRequest.setActions("testAction[n]"); // pick all test actions
+        TestTasksResponse response = tasksActions[randomIntBetween(0, nodesCount - 1)].execute(testTasksRequest).get();
+
+        // Get successful responses from all nodes except nodes that we filtered out
+        assertEquals(testNodes.length - filterNodes.size(), response.tasks.size());
+        assertEquals(0, response.getTaskFailures().size()); // no task failed
+        assertEquals(0, response.getNodeFailures().size()); // no nodes failed
+
+        // Make sure that filtered nodes didn't send any responses
+        for (TestTaskResponse taskResponse : response.tasks) {
+            String nodeId = taskResponse.getStatus();
+            assertFalse("Found response from filtered node " + nodeId, filterNodes.contains(nodeId));
+        }
+
+        // Release all node tasks and wait for response
+        checkLatch.countDown();
+        NodesResponse responses = future.get();
+        assertEquals(0, responses.failureCount());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/repositories/RepositoryBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/repositories/RepositoryBlocksIT.java
index 9c554da781a..503db65e810 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/repositories/RepositoryBlocksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/repositories/RepositoryBlocksIT.java
@@ -34,7 +34,7 @@ import static org.hamcrest.Matchers.hasSize;
 /**
  * This class tests that repository operations (Put, Delete, Verify) are blocked when the cluster is read-only.
  *
- * The @ClusterScope TEST is needed because this class updates the cluster setting "cluster.blocks.read_only".
+ * The @NodeScope TEST is needed because this class updates the cluster setting "cluster.blocks.read_only".
  */
 @ClusterScope(scope = ESIntegTestCase.Scope.TEST)
 public class RepositoryBlocksIT extends ESIntegTestCase {
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java
index f3a23be919d..82a2637d76b 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java
@@ -41,7 +41,7 @@ import static org.hamcrest.Matchers.hasSize;
 /**
  * This class tests that snapshot operations (Create, Delete, Restore) are blocked when the cluster is read-only.
  *
- * The @ClusterScope TEST is needed because this class updates the cluster setting "cluster.blocks.read_only".
+ * The @NodeScope TEST is needed because this class updates the cluster setting "cluster.blocks.read_only".
  */
 @ClusterScope(scope = ESIntegTestCase.Scope.TEST)
 public class SnapshotBlocksIT extends ESIntegTestCase {
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
index 1eeba76dbca..3b1040446e0 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.admin.indices.create;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.UnavailableShardsException;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
 import org.elasticsearch.action.search.SearchResponse;
@@ -29,6 +30,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.query.RangeQueryBuilder;
@@ -179,7 +181,6 @@ public class CreateIndexIT extends ESIntegTestCase {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/14932,https://github.com/elastic/elasticsearch/pull/15853" )
     public void testCreateAndDeleteIndexConcurrently() throws InterruptedException {
         createIndex("test");
         final AtomicInteger indexVersion = new AtomicInteger(0);
@@ -224,10 +225,14 @@ public class CreateIndexIT extends ESIntegTestCase {
         for (int i = 0; i < numDocs; i++) {
             try {
                 synchronized (indexVersionLock) {
-                    client().prepareIndex("test", "test").setSource("index_version", indexVersion.get()).get();
+                    client().prepareIndex("test", "test").setSource("index_version", indexVersion.get())
+                        .setTimeout(TimeValue.timeValueSeconds(10)).get();
                 }
             } catch (IndexNotFoundException inf) {
                 // fine
+            } catch (UnavailableShardsException ex) {
+                assertEquals(ex.getCause().getClass(), IndexNotFoundException.class);
+                // fine we run into a delete index while retrying
             }
         }
         latch.await();
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java
index 967a9026469..04892b82339 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.admin.indices.mapping.put;
 
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
 
 public class PutMappingRequestTests extends ESTestCase {
@@ -48,5 +49,12 @@ public class PutMappingRequestTests extends ESTestCase {
         r.source("somevalidmapping");
         ex = r.validate();
         assertNull("validation should succeed", ex);
+
+        r.setConcreteIndex(new Index("foo", "bar"));
+        ex = r.validate();
+        assertNotNull("source validation should fail", ex);
+        assertEquals(ex.getMessage(),
+            "Validation Failed: 1: either concrete index or unresolved indices can be set," +
+                " concrete index: [[foo/bar]] and indices: [myindex];");
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
index 2e39c39cfd2..c31993ebb81 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
@@ -32,6 +32,7 @@ import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.common.collect.ImmutableOpenIntMap;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
@@ -157,6 +158,7 @@ public class IndicesShardStoreRequestIT extends ESIntegTestCase {
                         .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "5")
                         .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false)
         ));
+
         indexRandomData(index);
         ensureGreen(index);
 
@@ -165,9 +167,10 @@ public class IndicesShardStoreRequestIT extends ESIntegTestCase {
 
         logger.info("--> corrupt random shard copies");
         Map<Integer, Set<String>> corruptedShardIDMap = new HashMap<>();
+        Index idx = resolveIndex(index);
         for (String node : internalCluster().nodesInclude(index)) {
             IndicesService indexServices = internalCluster().getInstance(IndicesService.class, node);
-            IndexService indexShards = indexServices.indexServiceSafe(index);
+            IndexService indexShards = indexServices.indexServiceSafe(idx);
             for (Integer shardId : indexShards.shardIds()) {
                 IndexShard shard = indexShards.getShard(shardId);
                 if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
index 13a2bb29981..77bd7c89927 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
@@ -99,7 +99,6 @@ public class IndicesStatsTests extends ESSingleNodeTestCase {
             assertThat(commitStats.getId(), notNullValue());
             assertThat(commitStats.getUserData(), hasKey(Translog.TRANSLOG_GENERATION_KEY));
             assertThat(commitStats.getUserData(), hasKey(Translog.TRANSLOG_UUID_KEY));
-
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/upgrade/UpgradeIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/upgrade/UpgradeIT.java
index baca9508a8b..620cef31f9a 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/upgrade/UpgradeIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/upgrade/UpgradeIT.java
@@ -134,7 +134,7 @@ public class UpgradeIT extends ESBackcompatTestCase {
         // means we can never generate ancient segments in this test (unless Lucene major version bumps but ES major version does not):
         assertFalse(hasAncientSegments(client(), indexToUpgrade));
 
-        logger.info("--> Running upgrade on index " + indexToUpgrade);
+        logger.info("--> Running upgrade on index {}", indexToUpgrade);
         assertNoFailures(client().admin().indices().prepareUpgrade(indexToUpgrade).get());
         awaitBusy(() -> {
             try {
@@ -228,7 +228,7 @@ public class UpgradeIT extends ESBackcompatTestCase {
         ESLogger logger = Loggers.getLogger(UpgradeIT.class);
         int toUpgrade = 0;
         for (IndexUpgradeStatus status : getUpgradeStatus(client, index)) {
-            logger.info("Index: " + status.getIndex() + ", total: " + status.getTotalBytes() + ", toUpgrade: " + status.getToUpgradeBytes());
+            logger.info("Index: {}, total: {}, toUpgrade: {}", status.getIndex(), status.getTotalBytes(), status.getToUpgradeBytes());
             toUpgrade += status.getToUpgradeBytes();
         }
         return toUpgrade == 0;
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
index 6cd877315cd..e36b4e4f028 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
@@ -28,19 +28,19 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.AutoCreateIndex;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.cluster.TestClusterService;
 import org.elasticsearch.test.transport.CapturingTransport;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
+import org.junit.After;
 import org.junit.Before;
 
 import java.nio.charset.StandardCharsets;
@@ -49,6 +49,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.LongSupplier;
 
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -57,16 +58,23 @@ import static org.mockito.Mockito.mock;
 public class TransportBulkActionTookTests extends ESTestCase {
 
     private ThreadPool threadPool;
+    private ClusterService clusterService;
 
     @Before
     public void setUp() throws Exception {
         super.setUp();
         threadPool = mock(ThreadPool.class);
+        clusterService = createClusterService(threadPool);
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        clusterService.close();
     }
 
     private TransportBulkAction createAction(boolean controlled, AtomicLong expected) {
         CapturingTransport capturingTransport = new CapturingTransport();
-        ClusterService clusterService = new TestClusterService(threadPool);
         TransportService transportService = new TransportService(capturingTransport, threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
@@ -191,7 +199,7 @@ public class TransportBulkActionTookTests extends ESTestCase {
         }
 
         @Override
-        public String[] concreteIndices(ClusterState state, IndicesRequest request) {
+        public String[] concreteIndexNames(ClusterState state, IndicesRequest request) {
             return request.indices();
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/IngestProxyActionFilterTests.java b/core/src/test/java/org/elasticsearch/action/ingest/IngestProxyActionFilterTests.java
index fa9728c4cd1..c027e4ecd18 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/IngestProxyActionFilterTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/IngestProxyActionFilterTests.java
@@ -29,10 +29,10 @@ import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.support.ActionFilterChain;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
index cf1cab2416b..f66dfa81eac 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
@@ -31,10 +31,8 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
 
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
-import java.util.List;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -46,7 +44,6 @@ public class SimulateExecutionServiceTests extends ESTestCase {
 
     private ThreadPool threadPool;
     private SimulateExecutionService executionService;
-    private Pipeline pipeline;
     private Processor processor;
     private IngestDocument ingestDocument;
 
@@ -59,7 +56,6 @@ public class SimulateExecutionServiceTests extends ESTestCase {
         );
         executionService = new SimulateExecutionService(threadPool);
         processor = new TestProcessor("id", "mock", ingestDocument -> {});
-        pipeline = new Pipeline("_id", "_description", new CompoundProcessor(processor, processor));
         ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
     }
 
@@ -68,74 +64,6 @@ public class SimulateExecutionServiceTests extends ESTestCase {
         threadPool.shutdown();
     }
 
-    public void testExecuteVerboseDocumentSimple() throws Exception {
-        List<SimulateProcessorResult> processorResultList = new ArrayList<>();
-        executionService.executeVerboseDocument(processor, ingestDocument, processorResultList);
-        SimulateProcessorResult result = new SimulateProcessorResult("id", ingestDocument);
-        assertThat(processorResultList.size(), equalTo(1));
-        assertThat(processorResultList.get(0).getProcessorTag(), equalTo(result.getProcessorTag()));
-        assertThat(processorResultList.get(0).getIngestDocument(), equalTo(result.getIngestDocument()));
-        assertThat(processorResultList.get(0).getFailure(), nullValue());
-    }
-
-    public void testExecuteVerboseDocumentSimpleException() throws Exception {
-        RuntimeException exception = new RuntimeException("mock_exception");
-        TestProcessor processor = new TestProcessor("id", "mock", ingestDocument -> { throw exception; });
-        List<SimulateProcessorResult> processorResultList = new ArrayList<>();
-        try {
-            executionService.executeVerboseDocument(processor, ingestDocument, processorResultList);
-            fail("should throw exception");
-        } catch (RuntimeException e) {
-            assertThat(e.getMessage(), equalTo("mock_exception"));
-        }
-        SimulateProcessorResult result = new SimulateProcessorResult("id", exception);
-        assertThat(processorResultList.size(), equalTo(1));
-        assertThat(processorResultList.get(0).getProcessorTag(), equalTo(result.getProcessorTag()));
-        assertThat(processorResultList.get(0).getFailure(), equalTo(result.getFailure()));
-    }
-
-    public void testExecuteVerboseDocumentCompoundSuccess() throws Exception {
-        TestProcessor processor1 = new TestProcessor("p1", "mock", ingestDocument -> { });
-        TestProcessor processor2 = new TestProcessor("p2", "mock", ingestDocument -> { });
-
-        Processor compoundProcessor = new CompoundProcessor(processor1, processor2);
-        List<SimulateProcessorResult> processorResultList = new ArrayList<>();
-        executionService.executeVerboseDocument(compoundProcessor, ingestDocument, processorResultList);
-        assertThat(processor1.getInvokedCounter(), equalTo(1));
-        assertThat(processor2.getInvokedCounter(), equalTo(1));
-        assertThat(processorResultList.size(), equalTo(2));
-        assertThat(processorResultList.get(0).getProcessorTag(), equalTo("p1"));
-        assertThat(processorResultList.get(0).getIngestDocument(), equalTo(ingestDocument));
-        assertThat(processorResultList.get(0).getFailure(), nullValue());
-        assertThat(processorResultList.get(1).getProcessorTag(), equalTo("p2"));
-        assertThat(processorResultList.get(1).getIngestDocument(), equalTo(ingestDocument));
-        assertThat(processorResultList.get(1).getFailure(), nullValue());
-    }
-
-    public void testExecuteVerboseDocumentCompoundOnFailure() throws Exception {
-        TestProcessor processor1 = new TestProcessor("p1", "mock", ingestDocument -> { });
-        TestProcessor processor2 = new TestProcessor("p2", "mock", ingestDocument -> { throw new RuntimeException("p2_exception"); });
-        TestProcessor onFailureProcessor1 = new TestProcessor("fail_p1", "mock", ingestDocument -> { });
-        TestProcessor onFailureProcessor2 = new TestProcessor("fail_p2", "mock", ingestDocument -> { throw new RuntimeException("fail_p2_exception"); });
-        TestProcessor onFailureProcessor3 = new TestProcessor("fail_p3", "mock", ingestDocument -> { });
-        CompoundProcessor onFailureCompoundProcessor = new CompoundProcessor(Collections.singletonList(onFailureProcessor2), Collections.singletonList(onFailureProcessor3));
-
-        Processor compoundProcessor = new CompoundProcessor(Arrays.asList(processor1, processor2), Arrays.asList(onFailureProcessor1, onFailureCompoundProcessor));
-        List<SimulateProcessorResult> processorResultList = new ArrayList<>();
-        executionService.executeVerboseDocument(compoundProcessor, ingestDocument, processorResultList);
-        assertThat(processor1.getInvokedCounter(), equalTo(1));
-        assertThat(processor2.getInvokedCounter(), equalTo(1));
-        assertThat(onFailureProcessor1.getInvokedCounter(), equalTo(1));
-        assertThat(onFailureProcessor2.getInvokedCounter(), equalTo(1));
-        assertThat(onFailureProcessor3.getInvokedCounter(), equalTo(1));
-        assertThat(processorResultList.size(), equalTo(5));
-        assertThat(processorResultList.get(0).getProcessorTag(), equalTo("p1"));
-        assertThat(processorResultList.get(1).getProcessorTag(), equalTo("p2"));
-        assertThat(processorResultList.get(2).getProcessorTag(), equalTo("fail_p1"));
-        assertThat(processorResultList.get(3).getProcessorTag(), equalTo("fail_p2"));
-        assertThat(processorResultList.get(4).getProcessorTag(), equalTo("fail_p3"));
-    }
-
     public void testExecuteVerboseItem() throws Exception {
         TestProcessor processor = new TestProcessor("test-id", "mock", ingestDocument -> {});
         Pipeline pipeline = new Pipeline("_id", "_description", new CompoundProcessor(processor, processor));
@@ -170,16 +98,43 @@ public class SimulateExecutionServiceTests extends ESTestCase {
         assertThat(simulateDocumentBaseResult.getFailure(), nullValue());
     }
 
-    public void testExecuteVerboseItemWithFailure() throws Exception {
+    public void testExecuteVerboseItemExceptionWithoutOnFailure() throws Exception {
+        TestProcessor processor1 = new TestProcessor("processor_0", "mock", ingestDocument -> {});
+        TestProcessor processor2 = new TestProcessor("processor_1", "mock", ingestDocument -> { throw new RuntimeException("processor failed"); });
+        TestProcessor processor3 = new TestProcessor("processor_2", "mock", ingestDocument -> {});
+        Pipeline pipeline = new Pipeline("_id", "_description", new CompoundProcessor(processor1, processor2, processor3));
+        SimulateDocumentResult actualItemResponse = executionService.executeDocument(pipeline, ingestDocument, true);
+        assertThat(processor1.getInvokedCounter(), equalTo(1));
+        assertThat(processor2.getInvokedCounter(), equalTo(1));
+        assertThat(processor3.getInvokedCounter(), equalTo(0));
+        assertThat(actualItemResponse, instanceOf(SimulateDocumentVerboseResult.class));
+        SimulateDocumentVerboseResult simulateDocumentVerboseResult = (SimulateDocumentVerboseResult) actualItemResponse;
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().size(), equalTo(2));
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getProcessorTag(), equalTo("processor_0"));
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getFailure(), nullValue());
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getIngestDocument(), not(sameInstance(ingestDocument)));
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getIngestDocument(), equalTo(ingestDocument));
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getIngestDocument().getSourceAndMetadata(), not(sameInstance(ingestDocument.getSourceAndMetadata())));
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getProcessorTag(), equalTo("processor_1"));
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getIngestDocument(), nullValue());
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getFailure(), instanceOf(RuntimeException.class));
+        RuntimeException runtimeException = (RuntimeException) simulateDocumentVerboseResult.getProcessorResults().get(1).getFailure();
+        assertThat(runtimeException.getMessage(), equalTo("processor failed"));
+    }
+
+    public void testExecuteVerboseItemWithOnFailure() throws Exception {
         TestProcessor processor1 = new TestProcessor("processor_0", "mock", ingestDocument -> { throw new RuntimeException("processor failed"); });
         TestProcessor processor2 = new TestProcessor("processor_1", "mock", ingestDocument -> {});
-        Pipeline pipeline = new Pipeline("_id", "_description", new CompoundProcessor(Collections.singletonList(processor1), Collections.singletonList(processor2)));
+        TestProcessor processor3 = new TestProcessor("processor_2", "mock", ingestDocument -> {});
+        Pipeline pipeline = new Pipeline("_id", "_description",
+                new CompoundProcessor(new CompoundProcessor(Collections.singletonList(processor1),
+                                Collections.singletonList(processor2)), processor3));
         SimulateDocumentResult actualItemResponse = executionService.executeDocument(pipeline, ingestDocument, true);
         assertThat(processor1.getInvokedCounter(), equalTo(1));
         assertThat(processor2.getInvokedCounter(), equalTo(1));
         assertThat(actualItemResponse, instanceOf(SimulateDocumentVerboseResult.class));
         SimulateDocumentVerboseResult simulateDocumentVerboseResult = (SimulateDocumentVerboseResult) actualItemResponse;
-        assertThat(simulateDocumentVerboseResult.getProcessorResults().size(), equalTo(2));
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().size(), equalTo(3));
         assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getProcessorTag(), equalTo("processor_0"));
         assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getIngestDocument(), nullValue());
         assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getFailure(), instanceOf(RuntimeException.class));
@@ -187,8 +142,20 @@ public class SimulateExecutionServiceTests extends ESTestCase {
         assertThat(runtimeException.getMessage(), equalTo("processor failed"));
         assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getProcessorTag(), equalTo("processor_1"));
         assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getIngestDocument(), not(sameInstance(ingestDocument)));
-        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getIngestDocument(), equalTo(ingestDocument));
+
+        IngestDocument ingestDocumentWithOnFailureMetadata = new IngestDocument(ingestDocument);
+        Map<String, String> metadata = ingestDocumentWithOnFailureMetadata.getIngestMetadata();
+        metadata.put(CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD, "mock");
+        metadata.put(CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD, "processor_0");
+        metadata.put(CompoundProcessor.ON_FAILURE_MESSAGE_FIELD, "processor failed");
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getIngestDocument(), equalTo(ingestDocumentWithOnFailureMetadata));
+
         assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getFailure(), nullValue());
+
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(2).getProcessorTag(), equalTo("processor_2"));
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(2).getIngestDocument(), not(sameInstance(ingestDocument)));
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(2).getIngestDocument(), equalTo(ingestDocument));
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(2).getFailure(), nullValue());
     }
 
     public void testExecuteItemWithFailure() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
index ed829d8fa8d..595b93773e3 100644
--- a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
@@ -45,7 +45,7 @@ public class MultiSearchRequestTests extends ESTestCase {
         IndicesQueriesRegistry registry = new IndicesQueriesRegistry(Settings.EMPTY, singletonMap("match_all", new MatchAllQueryParser()));
         byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/search/simple-msearch1.json");
         MultiSearchRequest request = RestMultiSearchAction.parseRequest(new MultiSearchRequest(), new BytesArray(data), false, null, null,
-                null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry, ParseFieldMatcher.EMPTY, null);
+                null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry, ParseFieldMatcher.EMPTY, null, null);
         assertThat(request.requests().size(), equalTo(8));
         assertThat(request.requests().get(0).indices()[0], equalTo("test"));
         assertThat(request.requests().get(0).indicesOptions(), equalTo(IndicesOptions.fromOptions(true, true, true, true, IndicesOptions.strictExpandOpenAndForbidClosed())));
@@ -72,7 +72,7 @@ public class MultiSearchRequestTests extends ESTestCase {
         IndicesQueriesRegistry registry = new IndicesQueriesRegistry(Settings.EMPTY, singletonMap("match_all", new MatchAllQueryParser()));
         byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/search/simple-msearch2.json");
         MultiSearchRequest request = RestMultiSearchAction.parseRequest(new MultiSearchRequest(), new BytesArray(data), false, null, null,
-                null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry, ParseFieldMatcher.EMPTY, null);
+                null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry, ParseFieldMatcher.EMPTY, null, null);
         assertThat(request.requests().size(), equalTo(5));
         assertThat(request.requests().get(0).indices()[0], equalTo("test"));
         assertThat(request.requests().get(0).types().length, equalTo(0));
@@ -91,7 +91,7 @@ public class MultiSearchRequestTests extends ESTestCase {
         IndicesQueriesRegistry registry = new IndicesQueriesRegistry(Settings.EMPTY, singletonMap("match_all", new MatchAllQueryParser()));
         byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/search/simple-msearch3.json");
         MultiSearchRequest request = RestMultiSearchAction.parseRequest(new MultiSearchRequest(), new BytesArray(data), false, null, null,
-                null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry, ParseFieldMatcher.EMPTY, null);
+                null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry, ParseFieldMatcher.EMPTY, null, null);
         assertThat(request.requests().size(), equalTo(4));
         assertThat(request.requests().get(0).indices()[0], equalTo("test0"));
         assertThat(request.requests().get(0).indices()[1], equalTo("test1"));
@@ -111,7 +111,7 @@ public class MultiSearchRequestTests extends ESTestCase {
         IndicesQueriesRegistry registry = new IndicesQueriesRegistry(Settings.EMPTY, singletonMap("match_all", new MatchAllQueryParser()));
         byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/search/simple-msearch4.json");
         MultiSearchRequest request = RestMultiSearchAction.parseRequest(new MultiSearchRequest(), new BytesArray(data), false, null, null,
-                null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry, ParseFieldMatcher.EMPTY, null);
+                null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry, ParseFieldMatcher.EMPTY, null, null);
         assertThat(request.requests().size(), equalTo(3));
         assertThat(request.requests().get(0).indices()[0], equalTo("test0"));
         assertThat(request.requests().get(0).indices()[1], equalTo("test1"));
@@ -133,7 +133,7 @@ public class MultiSearchRequestTests extends ESTestCase {
         IndicesQueriesRegistry registry = new IndicesQueriesRegistry(Settings.EMPTY, singletonMap("match_all", new MatchAllQueryParser()));
         byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/search/simple-msearch5.json");
         MultiSearchRequest request = RestMultiSearchAction.parseRequest(new MultiSearchRequest(), new BytesArray(data), true, null, null,
-                null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry, ParseFieldMatcher.EMPTY, null);
+                null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry, ParseFieldMatcher.EMPTY, null, null);
         assertThat(request.requests().size(), equalTo(3));
         assertThat(request.requests().get(0).indices()[0], equalTo("test0"));
         assertThat(request.requests().get(0).indices()[1], equalTo("test1"));
diff --git a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
index 2615e5a0b22..5d17232735f 100644
--- a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
@@ -34,7 +34,9 @@ import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.block.ClusterBlocks;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
@@ -44,6 +46,7 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
@@ -51,13 +54,13 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.cluster.TestClusterService;
 import org.elasticsearch.test.transport.CapturingTransport;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportResponseOptions;
 import org.elasticsearch.transport.TransportService;
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -73,6 +76,8 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.object.HasToString.hasToString;
@@ -83,7 +88,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
     private static final String TEST_CLUSTER = "test-cluster";
     private static ThreadPool THREAD_POOL;
 
-    private TestClusterService clusterService;
+    private ClusterService clusterService;
     private CapturingTransport transport;
 
     private TestTransportBroadcastByNodeAction action;
@@ -168,7 +173,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
         }
 
         @Override
-        public String[] concreteIndices(ClusterState state, IndicesRequest request) {
+        public String[] concreteIndexNames(ClusterState state, IndicesRequest request) {
             return request.indices();
         }
     }
@@ -182,7 +187,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
     public void setUp() throws Exception {
         super.setUp();
         transport = new CapturingTransport();
-        clusterService = new TestClusterService(THREAD_POOL);
+        clusterService = createClusterService(THREAD_POOL);
         final TransportService transportService = new TransportService(transport, THREAD_POOL);
         transportService.start();
         transportService.acceptIncomingRequests();
@@ -197,16 +202,24 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
         );
     }
 
-    void setClusterState(TestClusterService clusterService, String index) {
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        clusterService.close();
+    }
+
+    void setClusterState(ClusterService clusterService, String index) {
         int numberOfNodes = randomIntBetween(3, 5);
         DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
-        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(new Index(index,"_na_"));
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(new Index(index, "_na_"));
 
         int shardIndex = -1;
+        int totalIndexShards = 0;
         for (int i = 0; i < numberOfNodes; i++) {
             final DiscoveryNode node = newNode(i);
             discoBuilder = discoBuilder.put(node);
             int numberOfShards = randomIntBetween(1, 10);
+            totalIndexShards += numberOfShards;
             for (int j = 0; j < numberOfShards; j++) {
                 final ShardId shardId = new ShardId(index, "_na_", ++shardIndex);
                 ShardRouting shard = TestShardRouting.newShardRouting(index, shardId.getId(), node.id(), true, ShardRoutingState.STARTED);
@@ -219,9 +232,15 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
         discoBuilder.masterNodeId(newNode(numberOfNodes - 1).id());
         ClusterState.Builder stateBuilder = ClusterState.builder(new ClusterName(TEST_CLUSTER));
         stateBuilder.nodes(discoBuilder);
+        final IndexMetaData.Builder indexMetaData = IndexMetaData.builder(index)
+                .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+                .numberOfReplicas(0)
+                .numberOfShards(totalIndexShards);
+
+        stateBuilder.metaData(MetaData.builder().put(indexMetaData));
         stateBuilder.routingTable(RoutingTable.builder().add(indexRoutingTable.build()).build());
         ClusterState clusterState = stateBuilder.build();
-        clusterService.setState(clusterState);
+        setState(clusterService, clusterState);
     }
 
     static DiscoveryNode newNode(int nodeId) {
@@ -241,7 +260,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
 
         ClusterBlocks.Builder block = ClusterBlocks.builder()
                 .addGlobalBlock(new ClusterBlock(1, "test-block", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
-        clusterService.setState(ClusterState.builder(clusterService.state()).blocks(block));
+        setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block));
         try {
             action.new AsyncAction(null, request, listener).start();
             fail("expected ClusterBlockException");
@@ -256,7 +275,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
 
         ClusterBlocks.Builder block = ClusterBlocks.builder()
                 .addIndexBlock(TEST_INDEX, new ClusterBlock(1, "test-block", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
-        clusterService.setState(ClusterState.builder(clusterService.state()).blocks(block));
+        setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block));
         try {
             action.new AsyncAction(null, request, listener).start();
             fail("expected ClusterBlockException");
@@ -301,7 +320,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
         DiscoveryNodes.Builder builder = DiscoveryNodes.builder(clusterService.state().getNodes());
         builder.remove(masterNode.id());
 
-        clusterService.setState(ClusterState.builder(clusterService.state()).nodes(builder));
+        setState(clusterService, ClusterState.builder(clusterService.state()).nodes(builder));
 
         action.new AsyncAction(null, request, listener).start();
 
@@ -348,7 +367,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
 
         TransportResponse response = channel.getCapturedResponse();
         assertTrue(response instanceof TransportBroadcastByNodeAction.NodeResponse);
-        TransportBroadcastByNodeAction.NodeResponse nodeResponse = (TransportBroadcastByNodeAction.NodeResponse)response;
+        TransportBroadcastByNodeAction.NodeResponse nodeResponse = (TransportBroadcastByNodeAction.NodeResponse) response;
 
         // check the operation was executed on the correct node
         assertEquals("node id", nodeId, nodeResponse.getNodeId());
@@ -387,7 +406,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
             builder.remove(failedMasterNode.id());
             builder.masterNodeId(null);
 
-            clusterService.setState(ClusterState.builder(clusterService.state()).nodes(builder));
+            setState(clusterService, ClusterState.builder(clusterService.state()).nodes(builder));
         }
 
         action.new AsyncAction(null, request, listener).start();
diff --git a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
index 860f95ace55..226099e32f7 100644
--- a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
@@ -27,7 +27,6 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.NotMasterException;
 import org.elasticsearch.cluster.block.ClusterBlock;
@@ -36,6 +35,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
@@ -44,11 +44,11 @@ import org.elasticsearch.discovery.MasterNotDiscoveredException;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.cluster.TestClusterService;
 import org.elasticsearch.test.transport.CapturingTransport;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.TransportService;
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -58,13 +58,15 @@ import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
 public class TransportMasterNodeActionTests extends ESTestCase {
     private static ThreadPool threadPool;
 
-    private TestClusterService clusterService;
+    private ClusterService clusterService;
     private TransportService transportService;
     private CapturingTransport transport;
     private DiscoveryNode localNode;
@@ -81,13 +83,20 @@ public class TransportMasterNodeActionTests extends ESTestCase {
     public void setUp() throws Exception {
         super.setUp();
         transport = new CapturingTransport();
-        clusterService = new TestClusterService(threadPool);
+        clusterService = createClusterService(threadPool);
         transportService = new TransportService(transport, threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
         localNode = new DiscoveryNode("local_node", DummyTransportAddress.INSTANCE, Version.CURRENT);
         remoteNode = new DiscoveryNode("remote_node", DummyTransportAddress.INSTANCE, Version.CURRENT);
-        allNodes = new DiscoveryNode[] { localNode, remoteNode };
+        allNodes = new DiscoveryNode[]{localNode, remoteNode};
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        clusterService.close();
+        transportService.close();
     }
 
     @AfterClass
@@ -157,7 +166,7 @@ public class TransportMasterNodeActionTests extends ESTestCase {
         final Throwable exception = new Throwable();
         final Response response = new Response();
 
-        clusterService.setState(ClusterStateCreationUtils.state(localNode, localNode, allNodes));
+        setState(clusterService, ClusterStateCreationUtils.state(localNode, localNode, allNodes));
 
         new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool) {
             @Override
@@ -194,7 +203,7 @@ public class TransportMasterNodeActionTests extends ESTestCase {
                 randomFrom(RestStatus.values()), ClusterBlockLevel.ALL);
         ClusterState stateWithBlock = ClusterState.builder(ClusterStateCreationUtils.state(localNode, localNode, allNodes))
                 .blocks(ClusterBlocks.builder().addGlobalBlock(block)).build();
-        clusterService.setState(stateWithBlock);
+        setState(clusterService, stateWithBlock);
 
         new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool) {
             @Override
@@ -206,7 +215,7 @@ public class TransportMasterNodeActionTests extends ESTestCase {
 
         if (retryableBlock && unblockBeforeTimeout) {
             assertFalse(listener.isDone());
-            clusterService.setState(ClusterState.builder(ClusterStateCreationUtils.state(localNode, localNode, allNodes))
+            setState(clusterService, ClusterState.builder(ClusterStateCreationUtils.state(localNode, localNode, allNodes))
                     .blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK).build());
             assertTrue(listener.isDone());
             listener.get();
@@ -231,7 +240,7 @@ public class TransportMasterNodeActionTests extends ESTestCase {
         Request request = new Request();
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
 
-        clusterService.setState(ClusterStateCreationUtils.state(localNode, randomFrom(null, localNode, remoteNode), allNodes));
+        setState(clusterService, ClusterStateCreationUtils.state(localNode, randomFrom(null, localNode, remoteNode), allNodes));
 
         new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool) {
             @Override
@@ -246,7 +255,7 @@ public class TransportMasterNodeActionTests extends ESTestCase {
 
     public void testMasterNotAvailable() throws ExecutionException, InterruptedException {
         Request request = new Request().masterNodeTimeout(TimeValue.timeValueSeconds(0));
-        clusterService.setState(ClusterStateCreationUtils.state(localNode, null, allNodes));
+        setState(clusterService, ClusterStateCreationUtils.state(localNode, null, allNodes));
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool).execute(request, listener);
         assertTrue(listener.isDone());
@@ -255,18 +264,18 @@ public class TransportMasterNodeActionTests extends ESTestCase {
 
     public void testMasterBecomesAvailable() throws ExecutionException, InterruptedException {
         Request request = new Request();
-        clusterService.setState(ClusterStateCreationUtils.state(localNode, null, allNodes));
+        setState(clusterService, ClusterStateCreationUtils.state(localNode, null, allNodes));
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool).execute(request, listener);
         assertFalse(listener.isDone());
-        clusterService.setState(ClusterStateCreationUtils.state(localNode, localNode, allNodes));
+        setState(clusterService, ClusterStateCreationUtils.state(localNode, localNode, allNodes));
         assertTrue(listener.isDone());
         listener.get();
     }
 
     public void testDelegateToMaster() throws ExecutionException, InterruptedException {
         Request request = new Request();
-        clusterService.setState(ClusterStateCreationUtils.state(localNode, remoteNode, allNodes));
+        setState(clusterService, ClusterStateCreationUtils.state(localNode, remoteNode, allNodes));
 
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool).execute(request, listener);
@@ -286,7 +295,7 @@ public class TransportMasterNodeActionTests extends ESTestCase {
     public void testDelegateToFailingMaster() throws ExecutionException, InterruptedException {
         boolean failsWithConnectTransportException = randomBoolean();
         Request request = new Request().masterNodeTimeout(TimeValue.timeValueSeconds(failsWithConnectTransportException ? 60 : 0));
-        clusterService.setState(ClusterStateCreationUtils.state(localNode, remoteNode, allNodes));
+        setState(clusterService, ClusterStateCreationUtils.state(localNode, remoteNode, allNodes));
 
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool).execute(request, listener);
@@ -300,7 +309,7 @@ public class TransportMasterNodeActionTests extends ESTestCase {
         if (failsWithConnectTransportException) {
             transport.handleRemoteError(capturedRequest.requestId, new ConnectTransportException(remoteNode, "Fake error"));
             assertFalse(listener.isDone());
-            clusterService.setState(ClusterStateCreationUtils.state(localNode, localNode, allNodes));
+            setState(clusterService, ClusterStateCreationUtils.state(localNode, localNode, allNodes));
             assertTrue(listener.isDone());
             listener.get();
         } else {
@@ -322,13 +331,13 @@ public class TransportMasterNodeActionTests extends ESTestCase {
 
         final Response response = new Response();
 
-        clusterService.setState(ClusterStateCreationUtils.state(localNode, localNode, allNodes));
+        setState(clusterService, ClusterStateCreationUtils.state(localNode, localNode, allNodes));
 
         new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool) {
             @Override
             protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) throws Exception {
                 // The other node has become master, simulate failures of this node while publishing cluster state through ZenDiscovery
-                TransportMasterNodeActionTests.this.clusterService.setState(ClusterStateCreationUtils.state(localNode, remoteNode, allNodes));
+                setState(clusterService, ClusterStateCreationUtils.state(localNode, remoteNode, allNodes));
                 Throwable failure = randomBoolean()
                         ? new Discovery.FailedToCommitClusterStateException("Fake error")
                         : new NotMasterException("Fake error");
diff --git a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
index 0a6f94366f9..f41b540822e 100644
--- a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
@@ -24,17 +24,17 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeActionTests;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.cluster.TestClusterService;
 import org.elasticsearch.test.transport.CapturingTransport;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -50,12 +50,15 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+
 public class TransportNodesActionTests extends ESTestCase {
 
     private static ThreadPool THREAD_POOL;
     private static ClusterName CLUSTER_NAME = new ClusterName("test-cluster");
 
-    private TestClusterService clusterService;
+    private ClusterService clusterService;
     private CapturingTransport transport;
     private TestTransportNodesAction action;
 
@@ -114,7 +117,7 @@ public class TransportNodesActionTests extends ESTestCase {
     public void setUp() throws Exception {
         super.setUp();
         transport = new CapturingTransport();
-        clusterService = new TestClusterService(THREAD_POOL);
+        clusterService = createClusterService(THREAD_POOL);
         final TransportService transportService = new TransportService(transport, THREAD_POOL);
         transportService.start();
         transportService.acceptIncomingRequests();
@@ -142,7 +145,7 @@ public class TransportNodesActionTests extends ESTestCase {
         ClusterState.Builder stateBuilder = ClusterState.builder(CLUSTER_NAME);
         stateBuilder.nodes(discoBuilder);
         ClusterState clusterState = stateBuilder.build();
-        clusterService.setState(clusterState);
+        setState(clusterService, clusterState);
         action = new TestTransportNodesAction(
                 Settings.EMPTY,
                 THREAD_POOL,
@@ -155,6 +158,13 @@ public class TransportNodesActionTests extends ESTestCase {
         );
     }
 
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        clusterService.close();
+        transport.close();
+    }
+
     private static DiscoveryNode newNode(int nodeId, Map<String, String> attributes) {
         String node = "node_" + nodeId;
         return new DiscoveryNode(node, node, DummyTransportAddress.INSTANCE, attributes, Version.CURRENT);
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
index 03869974444..9170ff2e5a6 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
@@ -31,10 +31,10 @@ import org.elasticsearch.action.support.ActionFilter;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.broadcast.BroadcastRequest;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
@@ -43,10 +43,10 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.cluster.TestClusterService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.local.LocalTransport;
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -63,6 +63,8 @@ import java.util.concurrent.TimeUnit;
 import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.state;
 import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.stateWithAssignedPrimariesAndOneReplica;
 import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.stateWithNoShard;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -70,9 +72,8 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo;
 public class BroadcastReplicationTests extends ESTestCase {
 
     private static ThreadPool threadPool;
-    private TestClusterService clusterService;
+    private ClusterService clusterService;
     private TransportService transportService;
-    private LocalTransport transport;
     private TestBroadcastReplicationAction broadcastReplicationAction;
 
     @BeforeClass
@@ -84,14 +85,21 @@ public class BroadcastReplicationTests extends ESTestCase {
     @Before
     public void setUp() throws Exception {
         super.setUp();
-        transport = new LocalTransport(Settings.EMPTY, threadPool, Version.CURRENT, new NamedWriteableRegistry());
-        clusterService = new TestClusterService(threadPool);
+        LocalTransport transport = new LocalTransport(Settings.EMPTY, threadPool, Version.CURRENT, new NamedWriteableRegistry());
+        clusterService = createClusterService(threadPool);
         transportService = new TransportService(transport, threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
         broadcastReplicationAction = new TestBroadcastReplicationAction(Settings.EMPTY, threadPool, clusterService, transportService, new ActionFilters(new HashSet<ActionFilter>()), new IndexNameExpressionResolver(Settings.EMPTY), null);
     }
 
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        clusterService.close();
+        transportService.close();
+    }
+
     @AfterClass
     public static void afterClass() {
         ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
@@ -100,7 +108,7 @@ public class BroadcastReplicationTests extends ESTestCase {
 
     public void testNotStartedPrimary() throws InterruptedException, ExecutionException, IOException {
         final String index = "test";
-        clusterService.setState(state(index, randomBoolean(),
+        setState(clusterService, state(index, randomBoolean(),
                 randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.UNASSIGNED, ShardRoutingState.UNASSIGNED));
         logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
         Future<BroadcastResponse> response = (broadcastReplicationAction.execute(new DummyBroadcastRequest().indices(index)));
@@ -119,7 +127,7 @@ public class BroadcastReplicationTests extends ESTestCase {
 
     public void testStartedPrimary() throws InterruptedException, ExecutionException, IOException {
         final String index = "test";
-        clusterService.setState(state(index, randomBoolean(),
+        setState(clusterService, state(index, randomBoolean(),
                 ShardRoutingState.STARTED));
         logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
         Future<BroadcastResponse> response = (broadcastReplicationAction.execute(new DummyBroadcastRequest().indices(index)));
@@ -134,8 +142,8 @@ public class BroadcastReplicationTests extends ESTestCase {
 
     public void testResultCombine() throws InterruptedException, ExecutionException, IOException {
         final String index = "test";
-        int numShards = randomInt(3);
-        clusterService.setState(stateWithAssignedPrimariesAndOneReplica(index, numShards));
+        int numShards = 1 + randomInt(3);
+        setState(clusterService, stateWithAssignedPrimariesAndOneReplica(index, numShards));
         logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
         Future<BroadcastResponse> response = (broadcastReplicationAction.execute(new DummyBroadcastRequest().indices(index)));
         int succeeded = 0;
@@ -165,7 +173,7 @@ public class BroadcastReplicationTests extends ESTestCase {
     }
 
     public void testNoShards() throws InterruptedException, ExecutionException, IOException {
-        clusterService.setState(stateWithNoShard());
+        setState(clusterService, stateWithNoShard());
         logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
         BroadcastResponse response = executeAndAssertImmediateResponse(broadcastReplicationAction, new DummyBroadcastRequest());
         assertBroadcastResponse(0, 0, 0, response, null);
@@ -186,8 +194,8 @@ public class BroadcastReplicationTests extends ESTestCase {
         protected final Set<Tuple<ShardId, ActionListener<ReplicationResponse>>> capturedShardRequests = ConcurrentCollections.newConcurrentSet();
 
         public TestBroadcastReplicationAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
-                TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
-                TransportReplicationAction replicatedBroadcastShardAction) {
+                                              TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                              TransportReplicationAction replicatedBroadcastShardAction) {
             super("test-broadcast-replication-action", DummyBroadcastRequest::new, settings, threadPool, clusterService, transportService,
                     actionFilters, indexNameExpressionResolver, replicatedBroadcastShardAction);
         }
@@ -204,7 +212,7 @@ public class BroadcastReplicationTests extends ESTestCase {
 
         @Override
         protected BroadcastResponse newResponse(int successfulShards, int failedShards, int totalNumCopies,
-                List<ShardOperationFailedException> shardFailures) {
+                                                List<ShardOperationFailedException> shardFailures) {
             return new BroadcastResponse(totalNumCopies, successfulShards, failedShards, shardFailures);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
index 6b38d35c63b..bfe29229062 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
@@ -45,6 +45,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED;
 import static org.elasticsearch.test.ESTestCase.randomFrom;
+import static org.elasticsearch.test.ESTestCase.randomInt;
 import static org.elasticsearch.test.ESTestCase.randomIntBetween;
 
 /**
@@ -84,10 +85,11 @@ public class ClusterStateCreationUtils {
         }
         discoBuilder.localNodeId(newNode(0).id());
         discoBuilder.masterNodeId(newNode(1).id()); // we need a non-local master to test shard failures
+        final int primaryTerm = randomInt(200);
         IndexMetaData indexMetaData = IndexMetaData.builder(index).settings(Settings.builder()
                 .put(SETTING_VERSION_CREATED, Version.CURRENT)
                 .put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas)
-                .put(SETTING_CREATION_DATE, System.currentTimeMillis())).build();
+                .put(SETTING_CREATION_DATE, System.currentTimeMillis())).primaryTerm(0, primaryTerm).build();
 
         RoutingTable.Builder routing = new RoutingTable.Builder();
         routing.addAsNew(indexMetaData);
@@ -111,7 +113,8 @@ public class ClusterStateCreationUtils {
         } else {
             unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null);
         }
-        indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, 0, primaryNode, relocatingNode, null, true, primaryState, unassignedInfo));
+        indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, 0, primaryNode, relocatingNode, null, true,
+                primaryState, unassignedInfo));
 
         for (ShardRoutingState replicaState : replicaStates) {
             String replicaNode = null;
@@ -152,7 +155,7 @@ public class ClusterStateCreationUtils {
         discoBuilder.masterNodeId(newNode(1).id()); // we need a non-local master to test shard failures
         IndexMetaData indexMetaData = IndexMetaData.builder(index).settings(Settings.builder()
                 .put(SETTING_VERSION_CREATED, Version.CURRENT)
-                .put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 1)
+                .put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(SETTING_NUMBER_OF_REPLICAS, 1)
                 .put(SETTING_CREATION_DATE, System.currentTimeMillis())).build();
         ClusterState.Builder state = ClusterState.builder(new ClusterName("test"));
         state.nodes(discoBuilder);
@@ -163,8 +166,10 @@ public class ClusterStateCreationUtils {
             routing.addAsNew(indexMetaData);
             final ShardId shardId = new ShardId(index, "_na_", i);
             IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
-            indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, i, newNode(0).id(), null, null, true, ShardRoutingState.STARTED, null));
-            indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, i, newNode(1).id(), null, null, false, ShardRoutingState.STARTED, null));
+            indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, i, newNode(0).id(), null, null, true,
+                    ShardRoutingState.STARTED, null));
+            indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, i, newNode(1).id(), null, null, false,
+                    ShardRoutingState.STARTED, null));
             indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder.build());
         }
         state.routingTable(RoutingTable.builder().add(indexRoutingTableBuilder.build()).build());
@@ -229,12 +234,13 @@ public class ClusterStateCreationUtils {
 
     /**
      * Creates a cluster state where local node and master node can be specified
+     *
      * @param localNode  node in allNodes that is the local node
      * @param masterNode node in allNodes that is the master node. Can be null if no master exists
      * @param allNodes   all nodes in the cluster
      * @return cluster state
      */
-    public static  ClusterState state(DiscoveryNode localNode, DiscoveryNode masterNode, DiscoveryNode... allNodes) {
+    public static ClusterState state(DiscoveryNode localNode, DiscoveryNode masterNode, DiscoveryNode... allNodes) {
         DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
         for (DiscoveryNode node : allNodes) {
             discoBuilder.put(node);
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
index cbcfb6ab2b0..ba2695f11a0 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
@@ -26,7 +26,6 @@ import org.elasticsearch.action.WriteConsistencyLevel;
 import org.elasticsearch.action.support.ActionFilter;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.NotMasterException;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
@@ -44,6 +43,7 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -57,7 +57,6 @@ import org.elasticsearch.index.shard.ShardNotFoundException;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESAllocationTestCase;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.cluster.TestClusterService;
 import org.elasticsearch.test.transport.CapturingTransport;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportChannel;
@@ -65,6 +64,7 @@ import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportResponseOptions;
 import org.elasticsearch.transport.TransportService;
 import org.hamcrest.Matcher;
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -86,6 +86,8 @@ import java.util.function.Consumer;
 
 import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.state;
 import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.stateWithActivePrimary;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.Matchers.arrayWithSize;
@@ -102,7 +104,7 @@ public class TransportReplicationActionTests extends ESTestCase {
 
     private static ThreadPool threadPool;
 
-    private TestClusterService clusterService;
+    private ClusterService clusterService;
     private TransportService transportService;
     private CapturingTransport transport;
     private Action action;
@@ -121,7 +123,7 @@ public class TransportReplicationActionTests extends ESTestCase {
     public void setUp() throws Exception {
         super.setUp();
         transport = new CapturingTransport();
-        clusterService = new TestClusterService(threadPool);
+        clusterService = createClusterService(threadPool);
         transportService = new TransportService(transport, threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
@@ -129,6 +131,12 @@ public class TransportReplicationActionTests extends ESTestCase {
         count.set(1);
     }
 
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        clusterService.close();
+    }
+
     @AfterClass
     public static void afterClass() {
         ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
@@ -151,7 +159,7 @@ public class TransportReplicationActionTests extends ESTestCase {
 
         ClusterBlocks.Builder block = ClusterBlocks.builder()
                 .addGlobalBlock(new ClusterBlock(1, "non retryable", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
-        clusterService.setState(ClusterState.builder(clusterService.state()).blocks(block));
+        setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block));
         TransportReplicationAction.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener);
         reroutePhase.run();
         assertListenerThrows("primary phase should fail operation", listener, ClusterBlockException.class);
@@ -159,7 +167,7 @@ public class TransportReplicationActionTests extends ESTestCase {
 
         block = ClusterBlocks.builder()
                 .addGlobalBlock(new ClusterBlock(1, "retryable", true, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
-        clusterService.setState(ClusterState.builder(clusterService.state()).blocks(block));
+        setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block));
         listener = new PlainActionFuture<>();
         reroutePhase = action.new ReroutePhase(task, new Request().timeout("5ms"), listener);
         reroutePhase.run();
@@ -174,7 +182,7 @@ public class TransportReplicationActionTests extends ESTestCase {
 
         block = ClusterBlocks.builder()
                 .addGlobalBlock(new ClusterBlock(1, "non retryable", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
-        clusterService.setState(ClusterState.builder(clusterService.state()).blocks(block));
+        setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block));
         assertListenerThrows("primary phase should fail operation when moving from a retryable block to a non-retryable one", listener, ClusterBlockException.class);
         assertIndexShardUninitialized();
     }
@@ -187,7 +195,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         final String index = "test";
         final ShardId shardId = new ShardId(index, "_na_", 0);
         // no replicas in oder to skip the replication part
-        clusterService.setState(state(index, true,
+        setState(clusterService, state(index, true,
                 randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.UNASSIGNED));
         ReplicationTask task = maybeTask();
 
@@ -207,13 +215,13 @@ public class TransportReplicationActionTests extends ESTestCase {
         assertFalse("unassigned primary didn't cause a retry", listener.isDone());
         assertPhase(task, "waiting_for_retry");
 
-        clusterService.setState(state(index, true, ShardRoutingState.STARTED));
+        setState(clusterService, state(index, true, ShardRoutingState.STARTED));
         logger.debug("--> primary assigned state:\n{}", clusterService.state().prettyPrint());
 
         final IndexShardRoutingTable shardRoutingTable = clusterService.state().routingTable().index(index).shard(shardId.id());
         final String primaryNodeId = shardRoutingTable.primaryShard().currentNodeId();
         final List<CapturingTransport.CapturedRequest> capturedRequests =
-            transport.getCapturedRequestsByTargetNodeAndClear().get(primaryNodeId);
+                transport.getCapturedRequestsByTargetNodeAndClear().get(primaryNodeId);
         assertThat(capturedRequests, notNullValue());
         assertThat(capturedRequests.size(), equalTo(1));
         assertThat(capturedRequests.get(0).action, equalTo("testAction[p]"));
@@ -236,7 +244,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         ClusterState state = state(index, true, ShardRoutingState.RELOCATING);
         String relocationTargetNode = state.getRoutingTable().shardRoutingTable(shardId).primaryShard().relocatingNodeId();
         state = ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).localNodeId(relocationTargetNode)).build();
-        clusterService.setState(state);
+        setState(clusterService, state);
         logger.debug("--> relocation ongoing state:\n{}", clusterService.state().prettyPrint());
 
         Request request = new Request(shardId).timeout("1ms").routedBasedOnClusterVersion(clusterService.state().version() + 1);
@@ -257,13 +265,13 @@ public class TransportReplicationActionTests extends ESTestCase {
         RoutingAllocation.Result result = allocationService.applyStartedShards(state, Arrays.asList(relocationTarget));
         ClusterState updatedState = ClusterState.builder(clusterService.state()).routingResult(result).build();
 
-        clusterService.setState(updatedState);
+        setState(clusterService, updatedState);
         logger.debug("--> relocation complete state:\n{}", clusterService.state().prettyPrint());
 
         IndexShardRoutingTable shardRoutingTable = clusterService.state().routingTable().index(index).shard(shardId.id());
         final String primaryNodeId = shardRoutingTable.primaryShard().currentNodeId();
         final List<CapturingTransport.CapturedRequest> capturedRequests =
-            transport.getCapturedRequestsByTargetNodeAndClear().get(primaryNodeId);
+                transport.getCapturedRequestsByTargetNodeAndClear().get(primaryNodeId);
         assertThat(capturedRequests, notNullValue());
         assertThat(capturedRequests.size(), equalTo(1));
         assertThat(capturedRequests.get(0).action, equalTo("testAction[p]"));
@@ -273,7 +281,7 @@ public class TransportReplicationActionTests extends ESTestCase {
     public void testUnknownIndexOrShardOnReroute() throws InterruptedException {
         final String index = "test";
         // no replicas in oder to skip the replication part
-        clusterService.setState(state(index, true,
+        setState(clusterService, state(index, true,
                 randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.UNASSIGNED));
         logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
         Request request = new Request(new ShardId("unknown_index", "_na_", 0)).timeout("1ms");
@@ -296,7 +304,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         final ShardId shardId = new ShardId(index, "_na_", 0);
         ReplicationTask task = maybeTask();
 
-        clusterService.setState(stateWithActivePrimary(index, randomBoolean(), 3));
+        setState(clusterService, stateWithActivePrimary(index, randomBoolean(), 3));
         logger.debug("using state: \n{}", clusterService.state().prettyPrint());
 
         final IndexShardRoutingTable shardRoutingTable = clusterService.state().routingTable().index(index).shard(shardId.id());
@@ -325,7 +333,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         final String index = "test";
         final ShardId shardId = new ShardId(index, "_na_", 0);
         ClusterState state = stateWithActivePrimary(index, true, randomInt(5));
-        clusterService.setState(state);
+        setState(clusterService, state);
         Request request = new Request(shardId).timeout("1ms");
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         ReplicationTask task = maybeTask();
@@ -365,7 +373,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         String primaryTargetNodeId = state.getRoutingTable().shardRoutingTable(shardId).primaryShard().relocatingNodeId();
         // simulate execution of the primary phase on the relocation target node
         state = ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).localNodeId(primaryTargetNodeId)).build();
-        clusterService.setState(state);
+        setState(clusterService, state);
         Request request = new Request(shardId).timeout("1ms");
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         ReplicationTask task = maybeTask();
@@ -387,7 +395,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         final String index = "test";
         final ShardId shardId = new ShardId(index, "_na_", 0);
         // start with no replicas
-        clusterService.setState(stateWithActivePrimary(index, true, 0));
+        setState(clusterService, stateWithActivePrimary(index, true, 0));
         logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
         final ClusterState stateWithAddedReplicas = state(index, true, ShardRoutingState.STARTED, randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.STARTED);
         ReplicationTask task = maybeTask();
@@ -397,7 +405,7 @@ public class TransportReplicationActionTests extends ESTestCase {
             protected Tuple<Response, Request> shardOperationOnPrimary(MetaData metaData, Request shardRequest) throws Exception {
                 final Tuple<Response, Request> operationOnPrimary = super.shardOperationOnPrimary(metaData, shardRequest);
                 // add replicas after primary operation
-                ((TestClusterService) clusterService).setState(stateWithAddedReplicas);
+                setState(clusterService, stateWithAddedReplicas);
                 logger.debug("--> state after primary operation:\n{}", clusterService.state().prettyPrint());
                 return operationOnPrimary;
             }
@@ -422,7 +430,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         final String index = "test";
         final ShardId shardId = new ShardId(index, "_na_", 0);
         // start with a replica
-        clusterService.setState(state(index, true, ShardRoutingState.STARTED, randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.STARTED));
+        setState(clusterService, state(index, true, ShardRoutingState.STARTED, randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.STARTED));
         logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
         final ClusterState stateWithRelocatingReplica = state(index, true, ShardRoutingState.STARTED, ShardRoutingState.RELOCATING);
 
@@ -431,7 +439,7 @@ public class TransportReplicationActionTests extends ESTestCase {
             protected Tuple<Response, Request> shardOperationOnPrimary(MetaData metaData, Request shardRequest) throws Exception {
                 final Tuple<Response, Request> operationOnPrimary = super.shardOperationOnPrimary(metaData, shardRequest);
                 // set replica to relocating
-                ((TestClusterService) clusterService).setState(stateWithRelocatingReplica);
+                setState(clusterService, stateWithRelocatingReplica);
                 logger.debug("--> state after primary operation:\n{}", clusterService.state().prettyPrint());
                 return operationOnPrimary;
             }
@@ -447,7 +455,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         ShardRouting relocatingReplicaShard = stateWithRelocatingReplica.getRoutingTable().shardRoutingTable(index, shardId.id()).replicaShards().get(0);
         Map<String, List<CapturingTransport.CapturedRequest>> capturedRequestsByTargetNode = transport.getCapturedRequestsByTargetNodeAndClear();
         assertPhase(task, "replicating");
-        for (String node : new String[] {relocatingReplicaShard.currentNodeId(), relocatingReplicaShard.relocatingNodeId()}) {
+        for (String node : new String[]{relocatingReplicaShard.currentNodeId(), relocatingReplicaShard.relocatingNodeId()}) {
             List<CapturingTransport.CapturedRequest> requests = capturedRequestsByTargetNode.get(node);
             assertThat(requests, notNullValue());
             assertThat(requests.size(), equalTo(1));
@@ -458,7 +466,7 @@ public class TransportReplicationActionTests extends ESTestCase {
     public void testIndexDeletedAfterPrimaryOperation() {
         final String index = "test";
         final ShardId shardId = new ShardId(index, "_na_", 0);
-        clusterService.setState(state(index, true, ShardRoutingState.STARTED, ShardRoutingState.STARTED));
+        setState(clusterService, state(index, true, ShardRoutingState.STARTED, ShardRoutingState.STARTED));
         logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
         final ClusterState stateWithDeletedIndex = state(index + "_new", true, ShardRoutingState.STARTED, ShardRoutingState.RELOCATING);
 
@@ -467,7 +475,7 @@ public class TransportReplicationActionTests extends ESTestCase {
             protected Tuple<Response, Request> shardOperationOnPrimary(MetaData metaData, Request shardRequest) throws Exception {
                 final Tuple<Response, Request> operationOnPrimary = super.shardOperationOnPrimary(metaData, shardRequest);
                 // delete index after primary op
-                ((TestClusterService) clusterService).setState(stateWithDeletedIndex);
+                setState(clusterService, stateWithDeletedIndex);
                 logger.debug("--> state after primary operation:\n{}", clusterService.state().prettyPrint());
                 return operationOnPrimary;
             }
@@ -519,7 +527,7 @@ public class TransportReplicationActionTests extends ESTestCase {
             replicaStates[i] = ShardRoutingState.UNASSIGNED;
         }
 
-        clusterService.setState(state(index, true, ShardRoutingState.STARTED, replicaStates));
+        setState(clusterService, state(index, true, ShardRoutingState.STARTED, replicaStates));
         logger.debug("using consistency level of [{}], assigned shards [{}], total shards [{}]. expecting op to [{}]. using state: \n{}",
                 request.consistencyLevel(), 1 + assignedReplicas, 1 + assignedReplicas + unassignedReplicas, passesWriteConsistency ? "succeed" : "retry",
                 clusterService.state().prettyPrint());
@@ -547,7 +555,7 @@ public class TransportReplicationActionTests extends ESTestCase {
             for (int i = 0; i < replicaStates.length; i++) {
                 replicaStates[i] = ShardRoutingState.STARTED;
             }
-            clusterService.setState(state(index, true, ShardRoutingState.STARTED, replicaStates));
+            setState(clusterService, state(index, true, ShardRoutingState.STARTED, replicaStates));
             listener = new PlainActionFuture<>();
             primaryPhase = action.new PrimaryPhase(task, request, createTransportChannel(listener));
             primaryPhase.run();
@@ -567,7 +575,7 @@ public class TransportReplicationActionTests extends ESTestCase {
             // simulate execution of the replication phase on the relocation target node after relocation source was marked as relocated
             state = ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).localNodeId(primaryShard.relocatingNodeId())).build();
         }
-        clusterService.setState(state);
+        setState(clusterService, state);
 
         final IndexShardRoutingTable shardRoutingTable = clusterService.state().routingTable().index(index).shard(shardId.id());
         int assignedReplicas = 0;
@@ -602,7 +610,7 @@ public class TransportReplicationActionTests extends ESTestCase {
             // simulate execution of the primary phase on the relocation target node
             state = ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).localNodeId(primaryShard.relocatingNodeId())).build();
         }
-        clusterService.setState(state);
+        setState(clusterService, state);
 
         final IndexShardRoutingTable shardRoutingTable = clusterService.state().routingTable().index(index).shard(shardId.id());
         int assignedReplicas = 0;
@@ -622,11 +630,13 @@ public class TransportReplicationActionTests extends ESTestCase {
         final ShardIterator shardIt = shardRoutingTable.shardsIt();
         final ShardId shardId = shardIt.shardId();
         final Request request = new Request(shardId);
+        final long primaryTerm = randomInt(200);
+        request.primaryTerm(primaryTerm);
         final PlainActionFuture<Response> listener = new PlainActionFuture<>();
         ReplicationTask task = maybeTask();
         logger.debug("expecting [{}] assigned replicas, [{}] total shards. using state: \n{}", assignedReplicas, totalShards, clusterService.state().prettyPrint());
 
-        TransportReplicationAction.IndexShardReference reference = getOrCreateIndexShardOperationsCounter();
+        TransportReplicationAction.IndexShardReference reference = getOrCreateIndexShardOperationsCounter(0);
 
         ShardRouting primaryShard = state.getRoutingTable().shardRoutingTable(shardId).primaryShard();
         indexShardRouting.set(primaryShard);
@@ -646,7 +656,7 @@ public class TransportReplicationActionTests extends ESTestCase {
 
         HashMap<String, Request> nodesSentTo = new HashMap<>();
         boolean executeOnReplica =
-            action.shouldExecuteReplication(clusterService.state().getMetaData().index(shardId.getIndex()).getSettings());
+                action.shouldExecuteReplication(clusterService.state().getMetaData().index(shardId.getIndex()).getSettings());
         for (CapturingTransport.CapturedRequest capturedRequest : capturedRequests) {
             // no duplicate requests
             Request replicationRequest = (Request) capturedRequest.request;
@@ -703,7 +713,7 @@ public class TransportReplicationActionTests extends ESTestCase {
                     // get the shard the request was sent to
                     ShardRouting routing = clusterService.state().getRoutingNodes().node(capturedRequest.node.id()).get(request.shardId.id());
                     // and the shard that was requested to be failed
-                    ShardStateAction.ShardRoutingEntry shardRoutingEntry = (ShardStateAction.ShardRoutingEntry)shardFailedRequest.request;
+                    ShardStateAction.ShardRoutingEntry shardRoutingEntry = (ShardStateAction.ShardRoutingEntry) shardFailedRequest.request;
                     // the shard the request was sent to and the shard to be failed should be the same
                     assertEquals(shardRoutingEntry.getShardRouting(), routing);
                     failures.add(shardFailedRequest);
@@ -714,7 +724,7 @@ public class TransportReplicationActionTests extends ESTestCase {
                         CapturingTransport.CapturedRequest currentRequest = shardFailedRequest;
                         for (int retryNumber = 0; retryNumber < numberOfRetries; retryNumber++) {
                             // force a new cluster state to simulate a new master having been elected
-                            clusterService.setState(ClusterState.builder(clusterService.state()));
+                            setState(clusterService, ClusterState.builder(clusterService.state()));
                             transport.handleRemoteError(currentRequest.requestId, new NotMasterException("shard-failed-test"));
                             CapturingTransport.CapturedRequest[] retryRequests = transport.getCapturedRequestsAndClear();
                             assertEquals(1, retryRequests.length);
@@ -759,13 +769,16 @@ public class TransportReplicationActionTests extends ESTestCase {
         }
         // all replicas have responded so the counter should be decreased again
         assertIndexShardCounter(1);
+
+        // assert that nothing in the replica logic changes the primary term of the operation
+        assertThat(request.primaryTerm(), equalTo(primaryTerm));
     }
 
     public void testCounterOnPrimary() throws Exception {
         final String index = "test";
         final ShardId shardId = new ShardId(index, "_na_", 0);
         // no replica, we only want to test on primary
-        clusterService.setState(state(index, true, ShardRoutingState.STARTED));
+        setState(clusterService, state(index, true, ShardRoutingState.STARTED));
         logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
         Request request = new Request(shardId).timeout("100ms");
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
@@ -805,7 +818,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         final String index = "test";
         final ShardId shardId = new ShardId(index, "_na_", 0);
         // one replica to make sure replication is attempted
-        clusterService.setState(state(index, true,
+        setState(clusterService, state(index, true,
                 ShardRoutingState.STARTED, ShardRoutingState.STARTED));
         ShardRouting primaryShard = clusterService.state().routingTable().shardRoutingTable(shardId).primaryShard();
         indexShardRouting.set(primaryShard);
@@ -842,7 +855,7 @@ public class TransportReplicationActionTests extends ESTestCase {
 
     public void testReplicasCounter() throws Exception {
         final ShardId shardId = new ShardId("test", "_na_", 0);
-        clusterService.setState(state(shardId.getIndexName(), true,
+        setState(clusterService, state(shardId.getIndexName(), true,
                 ShardRoutingState.STARTED, ShardRoutingState.STARTED));
         action = new ActionWithDelay(Settings.EMPTY, "testActionWithExceptions", transportService, clusterService, threadPool);
         final Action.ReplicaOperationTransportHandler replicaOperationTransportHandler = action.new ReplicaOperationTransportHandler();
@@ -881,7 +894,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         action = new ActionWithExceptions(Settings.EMPTY, "testActionWithExceptions", transportService, clusterService, threadPool);
         final String index = "test";
         final ShardId shardId = new ShardId(index, "_na_", 0);
-        clusterService.setState(state(index, true,
+        setState(clusterService, state(index, true,
                 ShardRoutingState.STARTED, ShardRoutingState.STARTED));
         logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint());
         Request request = new Request(shardId).timeout("100ms");
@@ -901,7 +914,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         final String index = "test";
         final ShardId shardId = new ShardId(index, "_na_", 0);
         boolean localPrimary = true;
-        clusterService.setState(state(index, localPrimary,
+        setState(clusterService, state(index, localPrimary,
                 ShardRoutingState.STARTED, ShardRoutingState.STARTED));
         Action action = new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool) {
             @Override
@@ -953,8 +966,8 @@ public class TransportReplicationActionTests extends ESTestCase {
 
         // publish a new cluster state
         boolean localPrimaryOnRetry = randomBoolean();
-        clusterService.setState(state(index, localPrimaryOnRetry,
-            ShardRoutingState.STARTED, ShardRoutingState.STARTED));
+        setState(clusterService, state(index, localPrimaryOnRetry,
+                ShardRoutingState.STARTED, ShardRoutingState.STARTED));
         CapturingTransport.CapturedRequest[] primaryRetry = transport.getCapturedRequestsAndClear();
 
         // the request should be retried
@@ -981,7 +994,7 @@ public class TransportReplicationActionTests extends ESTestCase {
     /**
      * Returns testIndexShardOperationsCounter or initializes it if it was already created in this test run.
      */
-    private synchronized TransportReplicationAction.IndexShardReference getOrCreateIndexShardOperationsCounter() {
+    private synchronized TransportReplicationAction.IndexShardReference getOrCreateIndexShardOperationsCounter(long primaryTerm) {
         count.incrementAndGet();
         return new TransportReplicationAction.IndexShardReference() {
             @Override
@@ -1004,6 +1017,11 @@ public class TransportReplicationActionTests extends ESTestCase {
                 return shardRouting;
             }
 
+            @Override
+            public long opPrimaryTerm() {
+                return primaryTerm;
+            }
+
             @Override
             public void close() {
                 count.decrementAndGet();
@@ -1068,7 +1086,7 @@ public class TransportReplicationActionTests extends ESTestCase {
                ClusterService clusterService,
                ThreadPool threadPool) {
             super(settings, actionName, transportService, clusterService, null, threadPool,
-                    new ShardStateAction(settings, clusterService, transportService, null, null, threadPool), null,
+                    new ShardStateAction(settings, clusterService, transportService, null, null, threadPool),
                     new ActionFilters(new HashSet<ActionFilter>()), new IndexNameExpressionResolver(Settings.EMPTY), Request::new, Request::new, ThreadPool.Names.SAME);
         }
 
@@ -1099,13 +1117,15 @@ public class TransportReplicationActionTests extends ESTestCase {
             return false;
         }
 
+
         @Override
         protected IndexShardReference getIndexShardReferenceOnPrimary(ShardId shardId, Request request) {
-            return getOrCreateIndexShardOperationsCounter();
+            final IndexMetaData indexMetaData = clusterService.state().metaData().index(shardId.getIndex());
+            return getOrCreateIndexShardOperationsCounter(indexMetaData.primaryTerm(shardId.id()));
         }
 
-        protected IndexShardReference getIndexShardReferenceOnReplica(ShardId shardId, Request request) {
-            return getOrCreateIndexShardOperationsCounter();
+        protected IndexShardReference getIndexShardReferenceOnReplica(ShardId shardId, long opPrimaryTerm) {
+            return getOrCreateIndexShardOperationsCounter(opPrimaryTerm);
         }
     }
 
@@ -1193,7 +1213,8 @@ public class TransportReplicationActionTests extends ESTestCase {
      * Transport channel that is needed for replica operation testing.
      */
     public TransportChannel createTransportChannel(final PlainActionFuture<Response> listener) {
-        return createTransportChannel(listener, error -> {});
+        return createTransportChannel(listener, error -> {
+        });
     }
 
     public TransportChannel createTransportChannel(final PlainActionFuture<Response> listener, Consumer<Throwable> consumer) {
diff --git a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
index cf7b6745c8e..2dd31548cb9 100644
--- a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
@@ -36,17 +36,18 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.cluster.TestClusterService;
 import org.elasticsearch.test.transport.CapturingTransport;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportService;
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -59,13 +60,15 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.core.IsEqual.equalTo;
 
 public class TransportInstanceSingleOperationActionTests extends ESTestCase {
 
     private static ThreadPool THREAD_POOL;
 
-    private TestClusterService clusterService;
+    private ClusterService clusterService;
     private CapturingTransport transport;
     private TransportService transportService;
 
@@ -113,7 +116,7 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
 
         @Override
         protected ShardIterator shards(ClusterState clusterState, Request request) {
-            return clusterState.routingTable().index(request.concreteIndex()).shard(request.shardId).primaryShardIt();
+            return clusterState.routingTable().index(request.concreteIndex()).shard(request.shardId.getId()).primaryShardIt();
         }
     }
 
@@ -123,7 +126,7 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
         }
 
         @Override
-        public String[] concreteIndices(ClusterState state, IndicesRequest request) {
+        public String[] concreteIndexNames(ClusterState state, IndicesRequest request) {
             return request.indices();
         }
     }
@@ -137,7 +140,7 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
     public void setUp() throws Exception {
         super.setUp();
         transport = new CapturingTransport();
-        clusterService = new TestClusterService(THREAD_POOL);
+        clusterService = createClusterService(THREAD_POOL);
         transportService = new TransportService(transport, THREAD_POOL);
         transportService.start();
         transportService.acceptIncomingRequests();
@@ -151,6 +154,13 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
         );
     }
 
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        clusterService.close();
+        transportService.close();
+    }
+
     @AfterClass
     public static void destroyThreadPool() {
         ThreadPool.terminate(THREAD_POOL, 30, TimeUnit.SECONDS);
@@ -163,7 +173,7 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         ClusterBlocks.Builder block = ClusterBlocks.builder()
                 .addGlobalBlock(new ClusterBlock(1, "", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
-        clusterService.setState(ClusterState.builder(clusterService.state()).blocks(block));
+        setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block));
         try {
             action.new AsyncSingleAction(request, listener).start();
             listener.get();
@@ -178,9 +188,9 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
 
     public void testBasicRequestWorks() throws InterruptedException, ExecutionException, TimeoutException {
         Request request = new Request().index("test");
-        request.shardId = 0;
+        request.shardId = new ShardId("test", "_na_", 0);
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
-        clusterService.setState(ClusterStateCreationUtils.state("test", randomBoolean(), ShardRoutingState.STARTED));
+        setState(clusterService, ClusterStateCreationUtils.state("test", randomBoolean(), ShardRoutingState.STARTED));
         action.new AsyncSingleAction(request, listener).start();
         assertThat(transport.capturedRequests().length, equalTo(1));
         transport.handleResponse(transport.capturedRequests()[0].requestId, new Response());
@@ -189,9 +199,9 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
 
     public void testFailureWithoutRetry() throws Exception {
         Request request = new Request().index("test");
-        request.shardId = 0;
+        request.shardId = new ShardId("test", "_na_", 0);
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
-        clusterService.setState(ClusterStateCreationUtils.state("test", randomBoolean(), ShardRoutingState.STARTED));
+        setState(clusterService, ClusterStateCreationUtils.state("test", randomBoolean(), ShardRoutingState.STARTED));
 
         action.new AsyncSingleAction(request, listener).start();
         assertThat(transport.capturedRequests().length, equalTo(1));
@@ -215,14 +225,14 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
 
     public void testSuccessAfterRetryWithClusterStateUpdate() throws Exception {
         Request request = new Request().index("test");
-        request.shardId = 0;
+        request.shardId = new ShardId("test", "_na_", 0);
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         boolean local = randomBoolean();
-        clusterService.setState(ClusterStateCreationUtils.state("test", local, ShardRoutingState.INITIALIZING));
+        setState(clusterService, ClusterStateCreationUtils.state("test", local, ShardRoutingState.INITIALIZING));
         action.new AsyncSingleAction(request, listener).start();
         // this should fail because primary not initialized
         assertThat(transport.capturedRequests().length, equalTo(0));
-        clusterService.setState(ClusterStateCreationUtils.state("test", local, ShardRoutingState.STARTED));
+        setState(clusterService, ClusterStateCreationUtils.state("test", local, ShardRoutingState.STARTED));
         // this time it should work
         assertThat(transport.capturedRequests().length, equalTo(1));
         transport.handleResponse(transport.capturedRequests()[0].requestId, new Response());
@@ -231,10 +241,10 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
 
     public void testSuccessAfterRetryWithExceptionFromTransport() throws Exception {
         Request request = new Request().index("test");
-        request.shardId = 0;
+        request.shardId = new ShardId("test", "_na_", 0);
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         boolean local = randomBoolean();
-        clusterService.setState(ClusterStateCreationUtils.state("test", local, ShardRoutingState.STARTED));
+        setState(clusterService, ClusterStateCreationUtils.state("test", local, ShardRoutingState.STARTED));
         action.new AsyncSingleAction(request, listener).start();
         assertThat(transport.capturedRequests().length, equalTo(1));
         long requestId = transport.capturedRequests()[0].requestId;
@@ -242,7 +252,7 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
         DiscoveryNode node = clusterService.state().getNodes().getLocalNode();
         transport.handleLocalError(requestId, new ConnectTransportException(node, "test exception"));
         // trigger cluster state observer
-        clusterService.setState(ClusterStateCreationUtils.state("test", local, ShardRoutingState.STARTED));
+        setState(clusterService, ClusterStateCreationUtils.state("test", local, ShardRoutingState.STARTED));
         assertThat(transport.capturedRequests().length, equalTo(1));
         transport.handleResponse(transport.capturedRequests()[0].requestId, new Response());
         listener.get();
@@ -250,9 +260,9 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
 
     public void testRetryOfAnAlreadyTimedOutRequest() throws Exception {
         Request request = new Request().index("test").timeout(new TimeValue(0, TimeUnit.MILLISECONDS));
-        request.shardId = 0;
+        request.shardId = new ShardId("test", "_na_", 0);
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
-        clusterService.setState(ClusterStateCreationUtils.state("test", randomBoolean(), ShardRoutingState.STARTED));
+        setState(clusterService, ClusterStateCreationUtils.state("test", randomBoolean(), ShardRoutingState.STARTED));
         action.new AsyncSingleAction(request, listener).start();
         assertThat(transport.capturedRequests().length, equalTo(1));
         long requestId = transport.capturedRequests()[0].requestId;
@@ -299,9 +309,9 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
             }
         };
         Request request = new Request().index("test");
-        request.shardId = 0;
+        request.shardId = new ShardId("test", "_na_", 0);
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
-        clusterService.setState(ClusterStateCreationUtils.state("test", randomBoolean(), ShardRoutingState.STARTED));
+        setState(clusterService, ClusterStateCreationUtils.state("test", randomBoolean(), ShardRoutingState.STARTED));
         action.new AsyncSingleAction(request, listener).start();
         assertThat(transport.capturedRequests().length, equalTo(0));
         try {
diff --git a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java
index bcb26613388..d105a4bf63b 100644
--- a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.action.update;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -36,6 +37,7 @@ import org.elasticsearch.test.ESTestCase;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
@@ -179,4 +181,17 @@ public class UpdateRequestTests extends ESTestCase {
             assertThat(e.getMessage(), equalTo("Failed to derive xcontent"));
         }
     }
+
+    // Related to issue 15338
+    public void testFieldsParsing() throws Exception {
+        UpdateRequest request = new UpdateRequest("test", "type1", "1")
+                .source(new BytesArray("{\"doc\": {\"field1\": \"value1\"}, \"fields\": \"_source\"}"));
+        assertThat(request.doc().sourceAsMap().get("field1").toString(), equalTo("value1"));
+        assertThat(request.fields(), arrayContaining("_source"));
+
+        request = new UpdateRequest("test", "type2", "2")
+                .source(new BytesArray("{\"doc\": {\"field2\": \"value2\"}, \"fields\": [\"field1\", \"field2\"]}"));
+        assertThat(request.doc().sourceAsMap().get("field2").toString(), equalTo("value2"));
+        assertThat(request.fields(), arrayContaining("field1", "field2"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
index 3c269c39004..27f26e7c8c2 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.bootstrap;
 
+import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
@@ -31,6 +32,8 @@ import java.util.Set;
 import java.util.concurrent.atomic.AtomicLong;
 
 import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.not;
 
 public class BootstrapCheckTests extends ESTestCase {
 
@@ -79,9 +82,9 @@ public class BootstrapCheckTests extends ESTestCase {
 
     public void testFileDescriptorLimitsThrowsOnInvalidLimit() {
         final IllegalArgumentException e =
-                expectThrows(
-                        IllegalArgumentException.class,
-                        () -> new BootstrapCheck.FileDescriptorCheck(-randomIntBetween(0, Integer.MAX_VALUE)));
+            expectThrows(
+                IllegalArgumentException.class,
+                () -> new BootstrapCheck.FileDescriptorCheck(-randomIntBetween(0, Integer.MAX_VALUE)));
         assertThat(e.getMessage(), containsString("limit must be positive but was"));
     }
 
@@ -120,8 +123,8 @@ public class BootstrapCheckTests extends ESTestCase {
                     fail("should have failed due to memory not being locked");
                 } catch (final RuntimeException e) {
                     assertThat(
-                            e.getMessage(),
-                            containsString("memory locking requested for elasticsearch process but memory is not locked"));
+                        e.getMessage(),
+                        containsString("memory locking requested for elasticsearch process but memory is not locked"));
                 }
             } else {
                 // nothing should happen
@@ -157,6 +160,38 @@ public class BootstrapCheckTests extends ESTestCase {
         BootstrapCheck.check(true, Collections.singletonList(check));
     }
 
+    public void testMaxSizeVirtualMemory() {
+        final long rlimInfinity = Constants.MAC_OS_X ? 9223372036854775807L : -1L;
+        final AtomicLong maxSizeVirtualMemory = new AtomicLong(randomIntBetween(0, Integer.MAX_VALUE));
+        final BootstrapCheck.MaxSizeVirtualMemoryCheck check = new BootstrapCheck.MaxSizeVirtualMemoryCheck() {
+            @Override
+            long getMaxSizeVirtualMemory() {
+                return maxSizeVirtualMemory.get();
+            }
+
+            @Override
+            long getRlimInfinity() {
+                return rlimInfinity;
+            }
+        };
+
+        try {
+            BootstrapCheck.check(true, Collections.singletonList(check));
+            fail("should have failed due to max size virtual memory too low");
+        } catch (final RuntimeException e) {
+            assertThat(e.getMessage(), containsString("max size virtual memory"));
+        }
+
+        maxSizeVirtualMemory.set(rlimInfinity);
+
+        BootstrapCheck.check(true, Collections.singletonList(check));
+
+        // nothing should happen if max size virtual memory is not
+        // available
+        maxSizeVirtualMemory.set(Long.MIN_VALUE);
+        BootstrapCheck.check(true, Collections.singletonList(check));
+    }
+
     public void testEnforceLimits() {
         final Set<Setting> enforceSettings = BootstrapCheck.enforceSettings();
         final Setting setting = randomFrom(Arrays.asList(enforceSettings.toArray(new Setting[enforceSettings.size()])));
@@ -164,4 +199,12 @@ public class BootstrapCheckTests extends ESTestCase {
         assertTrue(BootstrapCheck.enforceLimits(settings));
     }
 
+    public void testMinMasterNodes() {
+        boolean isSet = randomBoolean();
+        BootstrapCheck.Check check = new BootstrapCheck.MinMasterNodesCheck(isSet);
+        assertThat(check.check(), not(equalTo(isSet)));
+        List<BootstrapCheck.Check> defaultChecks = BootstrapCheck.checks(Settings.EMPTY);
+
+        expectThrows(RuntimeException.class, () -> BootstrapCheck.check(true, defaultChecks));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java
new file mode 100644
index 00000000000..632646146fe
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import org.elasticsearch.Build;
+import org.elasticsearch.Version;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.monitor.jvm.JvmInfo;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.hasEntry;
+
+public class ElasticsearchCliTests extends ESTestCase {
+
+    public void testVersion() throws Exception {
+        runTestThatVersionIsMutuallyExclusiveToOtherOptions("-V", "-d");
+        runTestThatVersionIsMutuallyExclusiveToOtherOptions("-V", "--daemonize");
+        runTestThatVersionIsMutuallyExclusiveToOtherOptions("-V", "-p", "/tmp/pid");
+        runTestThatVersionIsMutuallyExclusiveToOtherOptions("-V", "--pidfile", "/tmp/pid");
+        runTestThatVersionIsMutuallyExclusiveToOtherOptions("--version", "-d");
+        runTestThatVersionIsMutuallyExclusiveToOtherOptions("--version", "--daemonize");
+        runTestThatVersionIsMutuallyExclusiveToOtherOptions("--version", "-p", "/tmp/pid");
+        runTestThatVersionIsMutuallyExclusiveToOtherOptions("--version", "--pidfile", "/tmp/pid");
+        runTestThatVersionIsReturned("-V");
+        runTestThatVersionIsReturned("--version");
+    }
+
+    private void runTestThatVersionIsMutuallyExclusiveToOtherOptions(String... args) throws Exception {
+        runTestVersion(
+                ExitCodes.USAGE,
+                output -> assertThat(
+                        output,
+                        containsString("ERROR: Elasticsearch version option is mutually exclusive with any other option")),
+                args);
+    }
+
+    private void runTestThatVersionIsReturned(String... args) throws Exception {
+        runTestVersion(ExitCodes.OK, output -> {
+            assertThat(output, containsString("Version: " + Version.CURRENT.toString()));
+            assertThat(output, containsString("Build: " + Build.CURRENT.shortHash() + "/" + Build.CURRENT.date()));
+            assertThat(output, containsString("JVM: " + JvmInfo.jvmInfo().version()));
+        }, args);
+    }
+
+    private void runTestVersion(int expectedStatus, Consumer<String> outputConsumer, String... args) throws Exception {
+        runTest(expectedStatus, false, outputConsumer, (foreground, pidFile, esSettings) -> {}, args);
+    }
+
+    public void testPositionalArgs() throws Exception {
+        runTest(
+            ExitCodes.USAGE,
+            false,
+            output -> assertThat(output, containsString("Positional arguments not allowed, found [foo]")),
+            (foreground, pidFile, esSettings) -> {},
+            "foo"
+        );
+        runTest(
+            ExitCodes.USAGE,
+            false,
+            output -> assertThat(output, containsString("Positional arguments not allowed, found [foo, bar]")),
+            (foreground, pidFile, esSettings) -> {},
+            "foo", "bar"
+        );
+        runTest(
+            ExitCodes.USAGE,
+            false,
+            output -> assertThat(output, containsString("Positional arguments not allowed, found [foo]")),
+            (foreground, pidFile, esSettings) -> {},
+            "-E", "something", "foo", "-E", "somethingelse"
+        );
+    }
+
+    public void testThatPidFileCanBeConfigured() throws Exception {
+        runPidFileTest(ExitCodes.USAGE, false, output -> assertThat(output, containsString("Option p/pidfile requires an argument")), "-p");
+        runPidFileTest(ExitCodes.OK, true, output -> {}, "-p", "/tmp/pid");
+        runPidFileTest(ExitCodes.OK, true, output -> {}, "--pidfile", "/tmp/pid");
+    }
+
+    private void runPidFileTest(final int expectedStatus, final boolean expectedInit, Consumer<String> outputConsumer, final String... args)
+            throws Exception {
+        runTest(
+                expectedStatus,
+                expectedInit,
+                outputConsumer,
+                (foreground, pidFile, esSettings) -> assertThat(pidFile, equalTo("/tmp/pid")),
+                args);
+    }
+
+    public void testThatParsingDaemonizeWorks() throws Exception {
+        runDaemonizeTest(true, "-d");
+        runDaemonizeTest(true, "--daemonize");
+        runDaemonizeTest(false);
+    }
+
+    private void runDaemonizeTest(final boolean expectedDaemonize, final String... args) throws Exception {
+        runTest(
+                ExitCodes.OK,
+                true,
+                output -> {},
+                (foreground, pidFile, esSettings) -> assertThat(foreground, equalTo(!expectedDaemonize)),
+                args);
+    }
+
+    public void testElasticsearchSettings() throws Exception {
+        runTest(
+                ExitCodes.OK,
+                true,
+                output -> {},
+                (foreground, pidFile, esSettings) -> {
+                    assertThat(esSettings.size(), equalTo(2));
+                    assertThat(esSettings, hasEntry("es.foo", "bar"));
+                    assertThat(esSettings, hasEntry("es.baz", "qux"));
+                },
+                "-Ees.foo=bar", "-E", "es.baz=qux"
+        );
+    }
+
+    public void testElasticsearchSettingPrefix() throws Exception {
+        runElasticsearchSettingPrefixTest("-E", "foo");
+        runElasticsearchSettingPrefixTest("-E", "foo=bar");
+        runElasticsearchSettingPrefixTest("-E", "=bar");
+    }
+
+    private void runElasticsearchSettingPrefixTest(String... args) throws Exception {
+        runTest(
+                ExitCodes.USAGE,
+                false,
+                output -> assertThat(output, containsString("Elasticsearch settings must be prefixed with [es.] but was [")),
+                (foreground, pidFile, esSettings) -> {},
+                args
+        );
+    }
+
+    public void testElasticsearchSettingCanNotBeEmpty() throws Exception {
+        runTest(
+                ExitCodes.USAGE,
+                false,
+                output -> assertThat(output, containsString("Elasticsearch setting [es.foo] must not be empty")),
+                (foreground, pidFile, esSettings) -> {},
+                "-E", "es.foo="
+        );
+    }
+
+    public void testUnknownOption() throws Exception {
+        runTest(
+                ExitCodes.USAGE,
+                false,
+                output -> assertThat(output, containsString("network.host is not a recognized option")),
+                (foreground, pidFile, esSettings) -> {},
+                "--network.host");
+    }
+
+    private interface InitConsumer {
+        void accept(final boolean foreground, final String pidFile, final Map<String, String> esSettings);
+    }
+
+    private void runTest(
+            final int expectedStatus,
+            final boolean expectedInit,
+            final Consumer<String> outputConsumer,
+            final InitConsumer initConsumer,
+            String... args) throws Exception {
+        final MockTerminal terminal = new MockTerminal();
+        try {
+            final AtomicBoolean init = new AtomicBoolean();
+            final int status = Elasticsearch.main(args, new Elasticsearch() {
+                @Override
+                void init(final boolean daemonize, final String pidFile, final Map<String, String> esSettings) {
+                    init.set(true);
+                    initConsumer.accept(!daemonize, pidFile, esSettings);
+                }
+            }, terminal);
+            assertThat(status, equalTo(expectedStatus));
+            assertThat(init.get(), equalTo(expectedInit));
+            outputConsumer.accept(terminal.getOutput());
+        } catch (Throwable t) {
+            // if an unexpected exception is thrown, we log
+            // terminal output to aid debugging
+            logger.info(terminal.getOutput());
+            // rethrow so the test fails
+            throw t;
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/BasicBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/BasicBackwardsCompatibilityIT.java
index 7e46825398b..ae739701593 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/BasicBackwardsCompatibilityIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/BasicBackwardsCompatibilityIT.java
@@ -191,7 +191,7 @@ public class BasicBackwardsCompatibilityIT extends ESBackcompatTestCase {
             numDocs *= 2;
         }
 
-        logger.info(" --> waiting for relocation to complete", numDocs);
+        logger.info(" --> waiting for relocation of [{}] docs to complete", numDocs);
         ensureYellow("test");// move all shards to the new node (it waits on relocation)
         final int numIters = randomIntBetween(10, 20);
         for (int i = 0; i < numIters; i++) {
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java
index 8e3dbd5f563..784813c3b4e 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java
@@ -41,6 +41,7 @@ import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.IndexFolderUpgrader;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -105,6 +106,8 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
 
     List<String> indexes;
     List<String> unsupportedIndexes;
+    static String singleDataPathNodeName;
+    static String multiDataPathNodeName;
     static Path singleDataPath;
     static Path[] multiDataPath;
 
@@ -127,6 +130,8 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
 
     @AfterClass
     public static void tearDownStatics() {
+        singleDataPathNodeName = null;
+        multiDataPathNodeName = null;
         singleDataPath = null;
         multiDataPath = null;
     }
@@ -157,15 +162,17 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
         InternalTestCluster.Async<String> multiDataPathNode = internalCluster().startNodeAsync(nodeSettings.build());
 
         // find single data path dir
-        Path[] nodePaths = internalCluster().getInstance(NodeEnvironment.class, singleDataPathNode.get()).nodeDataPaths();
+        singleDataPathNodeName = singleDataPathNode.get();
+        Path[] nodePaths = internalCluster().getInstance(NodeEnvironment.class, singleDataPathNodeName).nodeDataPaths();
         assertEquals(1, nodePaths.length);
         singleDataPath = nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER);
         assertFalse(Files.exists(singleDataPath));
         Files.createDirectories(singleDataPath);
-        logger.info("--> Single data path: " + singleDataPath.toString());
+        logger.info("--> Single data path: {}", singleDataPath);
 
         // find multi data path dirs
-        nodePaths = internalCluster().getInstance(NodeEnvironment.class, multiDataPathNode.get()).nodeDataPaths();
+        multiDataPathNodeName = multiDataPathNode.get();
+        nodePaths = internalCluster().getInstance(NodeEnvironment.class, multiDataPathNodeName).nodeDataPaths();
         assertEquals(2, nodePaths.length);
         multiDataPath = new Path[] {nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER),
                                    nodePaths[1].resolve(NodeEnvironment.INDICES_FOLDER)};
@@ -173,11 +180,18 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
         assertFalse(Files.exists(multiDataPath[1]));
         Files.createDirectories(multiDataPath[0]);
         Files.createDirectories(multiDataPath[1]);
-        logger.info("--> Multi data paths: " + multiDataPath[0].toString() + ", " + multiDataPath[1].toString());
+        logger.info("--> Multi data paths: {}, {}", multiDataPath[0], multiDataPath[1]);
 
         replicas.get(); // wait for replicas
     }
 
+    void upgradeIndexFolder() throws Exception {
+        final NodeEnvironment nodeEnvironment = internalCluster().getInstance(NodeEnvironment.class, singleDataPathNodeName);
+        IndexFolderUpgrader.upgradeIndicesIfNeeded(Settings.EMPTY, nodeEnvironment);
+        final NodeEnvironment nodeEnv = internalCluster().getInstance(NodeEnvironment.class, multiDataPathNodeName);
+        IndexFolderUpgrader.upgradeIndicesIfNeeded(Settings.EMPTY, nodeEnv);
+    }
+
     String loadIndex(String indexFile) throws Exception {
         Path unzipDir = createTempDir();
         Path unzipDataDir = unzipDir.resolve("data");
@@ -239,13 +253,13 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                 if (file.getFileName().toString().equals(IndexWriter.WRITE_LOCK_NAME)) {
                     // skip lock file, we don't need it
-                    logger.trace("Skipping lock file: " + file.toString());
+                    logger.trace("Skipping lock file: {}", file);
                     return FileVisitResult.CONTINUE;
                 }
 
                 Path relativeFile = src.relativize(file);
                 Path destFile = destinationDataPath.resolve(indexName).resolve(relativeFile);
-                logger.trace("--> Moving " + relativeFile.toString() + " to " + destFile.toString());
+                logger.trace("--> Moving {} to {}", relativeFile, destFile);
                 Files.move(file, destFile);
                 assertFalse(Files.exists(file));
                 assertTrue(Files.exists(destFile));
@@ -269,7 +283,7 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
 
         for (String index : indexes) {
             if (expectedVersions.remove(index) == false) {
-                logger.warn("Old indexes tests contain extra index: " + index);
+                logger.warn("Old indexes tests contain extra index: {}", index);
             }
         }
         if (expectedVersions.isEmpty() == false) {
@@ -287,15 +301,19 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
         Collections.shuffle(indexes, random());
         for (String index : indexes) {
             long startTime = System.currentTimeMillis();
-            logger.info("--> Testing old index " + index);
+            logger.info("--> Testing old index {}", index);
             assertOldIndexWorks(index);
-            logger.info("--> Done testing " + index + ", took " + ((System.currentTimeMillis() - startTime) / 1000.0) + " seconds");
+            logger.info("--> Done testing {}, took {} seconds", index, (System.currentTimeMillis() - startTime) / 1000.0);
         }
     }
 
     void assertOldIndexWorks(String index) throws Exception {
         Version version = extractVersion(index);
         String indexName = loadIndex(index);
+        // we explicitly upgrade the index folders as these indices
+        // are imported as dangling indices and not available on
+        // node startup
+        upgradeIndexFolder();
         importIndex(indexName);
         assertIndexSanity(indexName, version);
         assertBasicSearchWorks(indexName);
@@ -344,7 +362,7 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
         SearchResponse searchRsp = searchReq.get();
         ElasticsearchAssertions.assertNoFailures(searchRsp);
         long numDocs = searchRsp.getHits().getTotalHits();
-        logger.info("Found " + numDocs + " in old index");
+        logger.info("Found {} in old index", numDocs);
 
         logger.info("--> testing basic search with sort");
         searchReq.addSort("long_sort", SortOrder.ASC);
@@ -523,7 +541,7 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
         for (String indexFile : indexes) {
             String indexName = indexFile.replace(".zip", "").toLowerCase(Locale.ROOT).replace("unsupported-", "index-");
             Path nodeDir = getNodeDir(indexFile);
-            logger.info("Parsing cluster state files from index [" + indexName + "]");
+            logger.info("Parsing cluster state files from index [{}]", indexName);
             assertNotNull(globalFormat.loadLatestState(logger, nodeDir)); // no exception
             Path indexDir = nodeDir.resolve("indices").resolve(indexName);
             assertNotNull(indexFormat.loadLatestState(logger, indexDir)); // no exception
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java
index 23163b86112..9fe83f65c45 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java
@@ -28,7 +28,7 @@ public class RecoveryWithUnsupportedIndicesIT extends StaticIndexBackwardCompati
     public void testUpgradeStartClusterOn_0_20_6() throws Exception {
         String indexName = "unsupported-0.20.6";
 
-        logger.info("Checking static index " + indexName);
+        logger.info("Checking static index {}", indexName);
         Settings nodeSettings = prepareBackwardsDataDir(getBwcIndicesPath().resolve(indexName + ".zip"), NetworkModule.HTTP_ENABLED.getKey(), true);
         try {
             internalCluster().startNode(nodeSettings);
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
index 483040209d0..5b81621e6dd 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
@@ -108,7 +108,7 @@ public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase {
 
         for (String repoVersion : repoVersions) {
             if (expectedVersions.remove(repoVersion) == false) {
-                logger.warn("Old repositories tests contain extra repo: " + repoVersion);
+                logger.warn("Old repositories tests contain extra repo: {}", repoVersion);
             }
         }
         if (expectedVersions.isEmpty() == false) {
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/StaticIndexBackwardCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/StaticIndexBackwardCompatibilityIT.java
index 794aea85487..3884d3475e1 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/StaticIndexBackwardCompatibilityIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/StaticIndexBackwardCompatibilityIT.java
@@ -36,7 +36,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 public class StaticIndexBackwardCompatibilityIT extends ESIntegTestCase {
 
     public void loadIndex(String index, Object... settings) throws Exception {
-        logger.info("Checking static index " + index);
+        logger.info("Checking static index {}", index);
         Settings nodeSettings = prepareBackwardsDataDir(getDataPath(index + ".zip"), settings);
         internalCluster().startNode(nodeSettings);
         ensureGreen(index);
diff --git a/core/src/test/java/org/elasticsearch/cli/CommandTests.java b/core/src/test/java/org/elasticsearch/cli/CommandTests.java
new file mode 100644
index 00000000000..153bd4600b9
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cli/CommandTests.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cli;
+
+import joptsimple.OptionSet;
+import org.elasticsearch.test.ESTestCase;
+
+public class CommandTests extends ESTestCase {
+
+    static class UserErrorCommand extends Command {
+        UserErrorCommand() {
+            super("Throws a user error");
+        }
+        @Override
+        protected void execute(Terminal terminal, OptionSet options) throws Exception {
+            throw new UserError(ExitCodes.DATA_ERROR, "Bad input");
+        }
+    }
+
+    static class NoopCommand extends Command {
+        boolean executed = false;
+        NoopCommand() {
+            super("Does nothing");
+        }
+        @Override
+        protected void execute(Terminal terminal, OptionSet options) throws Exception {
+            terminal.println("Normal output");
+            terminal.println(Terminal.Verbosity.SILENT, "Silent output");
+            terminal.println(Terminal.Verbosity.VERBOSE, "Verbose output");
+            executed = true;
+        }
+        @Override
+        protected void printAdditionalHelp(Terminal terminal) {
+            terminal.println("Some extra help");
+        }
+    }
+
+    public void testHelp() throws Exception {
+        NoopCommand command = new NoopCommand();
+        MockTerminal terminal = new MockTerminal();
+        String[] args = {"-h"};
+        int status = command.main(args, terminal);
+        String output = terminal.getOutput();
+        assertEquals(output, ExitCodes.OK, status);
+        assertTrue(output, output.contains("Does nothing"));
+        assertTrue(output, output.contains("Some extra help"));
+        assertFalse(command.executed);
+
+        command = new NoopCommand();
+        String[] args2 = {"--help"};
+        status = command.main(args2, terminal);
+        output = terminal.getOutput();
+        assertEquals(output, ExitCodes.OK, status);
+        assertTrue(output, output.contains("Does nothing"));
+        assertTrue(output, output.contains("Some extra help"));
+        assertFalse(command.executed);
+    }
+
+    public void testVerbositySilentAndVerbose() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        NoopCommand command = new NoopCommand();
+        String[] args = {"-v", "-s"};
+        UserError e = expectThrows(UserError.class, () -> {
+            command.mainWithoutErrorHandling(args, terminal);
+        });
+        assertTrue(e.getMessage(), e.getMessage().contains("Cannot specify -s and -v together"));
+    }
+
+    public void testSilentVerbosity() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        NoopCommand command = new NoopCommand();
+        String[] args = {"-s"};
+        command.main(args, terminal);
+        String output = terminal.getOutput();
+        assertTrue(output, output.contains("Silent output"));
+    }
+
+    public void testNormalVerbosity() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        terminal.setVerbosity(Terminal.Verbosity.SILENT);
+        NoopCommand command = new NoopCommand();
+        String[] args = {};
+        command.main(args, terminal);
+        String output = terminal.getOutput();
+        assertTrue(output, output.contains("Normal output"));
+    }
+
+    public void testVerboseVerbosity() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        NoopCommand command = new NoopCommand();
+        String[] args = {"-v"};
+        command.main(args, terminal);
+        String output = terminal.getOutput();
+        assertTrue(output, output.contains("Verbose output"));
+    }
+
+    public void testUserError() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        UserErrorCommand command = new UserErrorCommand();
+        String[] args = {};
+        int status = command.main(args, terminal);
+        String output = terminal.getOutput();
+        assertEquals(output, ExitCodes.DATA_ERROR, status);
+        assertTrue(output, output.contains("ERROR: Bad input"));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cli/MultiCommandTests.java b/core/src/test/java/org/elasticsearch/cli/MultiCommandTests.java
new file mode 100644
index 00000000000..4f91d378440
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cli/MultiCommandTests.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cli;
+
+import joptsimple.OptionSet;
+import org.junit.Before;
+
+public class MultiCommandTests extends CommandTestCase {
+
+    static class DummyMultiCommand extends MultiCommand {
+        DummyMultiCommand() {
+            super("A dummy multi command");
+        }
+    }
+
+    static class DummySubCommand extends Command {
+        DummySubCommand() {
+            super("A dummy subcommand");
+        }
+        @Override
+        protected void execute(Terminal terminal, OptionSet options) throws Exception {
+            terminal.println("Arguments: " + options.nonOptionArguments().toString());
+        }
+    }
+
+    DummyMultiCommand multiCommand;
+
+    @Before
+    public void setupCommand() {
+        multiCommand = new DummyMultiCommand();
+    }
+
+    @Override
+    protected Command newCommand() {
+        return multiCommand;
+    }
+
+    public void testNoCommandsConfigured() throws Exception {
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> {
+            execute();
+        });
+        assertEquals("No subcommands configured", e.getMessage());
+    }
+
+    public void testUnknownCommand() throws Exception {
+        multiCommand.subcommands.put("something", new DummySubCommand());
+        UserError e = expectThrows(UserError.class, () -> {
+            execute("somethingelse");
+        });
+        assertEquals(ExitCodes.USAGE, e.exitCode);
+        assertEquals("Unknown command [somethingelse]", e.getMessage());
+    }
+
+    public void testMissingCommand() throws Exception {
+        multiCommand.subcommands.put("command1", new DummySubCommand());
+        UserError e = expectThrows(UserError.class, () -> {
+            execute();
+        });
+        assertEquals(ExitCodes.USAGE, e.exitCode);
+        assertEquals("Missing command", e.getMessage());
+    }
+
+    public void testHelp() throws Exception {
+        multiCommand.subcommands.put("command1", new DummySubCommand());
+        multiCommand.subcommands.put("command2", new DummySubCommand());
+        execute("-h");
+        String output = terminal.getOutput();
+        assertTrue(output, output.contains("command1"));
+        assertTrue(output, output.contains("command2"));
+    }
+
+    public void testSubcommandHelp() throws Exception {
+        multiCommand.subcommands.put("command1", new DummySubCommand());
+        multiCommand.subcommands.put("command2", new DummySubCommand());
+        execute("command2", "-h");
+        String output = terminal.getOutput();
+        assertFalse(output, output.contains("command1"));
+        assertTrue(output, output.contains("A dummy subcommand"));
+    }
+
+    public void testSubcommandArguments() throws Exception {
+        multiCommand.subcommands.put("command1", new DummySubCommand());
+        execute("command1", "foo", "bar");
+        String output = terminal.getOutput();
+        assertFalse(output, output.contains("command1"));
+        assertTrue(output, output.contains("Arguments: [foo, bar]"));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/cli/TerminalTests.java b/core/src/test/java/org/elasticsearch/cli/TerminalTests.java
similarity index 94%
rename from core/src/test/java/org/elasticsearch/common/cli/TerminalTests.java
rename to core/src/test/java/org/elasticsearch/cli/TerminalTests.java
index deb64e906b4..6673bdbc858 100644
--- a/core/src/test/java/org/elasticsearch/common/cli/TerminalTests.java
+++ b/core/src/test/java/org/elasticsearch/cli/TerminalTests.java
@@ -17,9 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.common.cli;
+package org.elasticsearch.cli;
 
-public class TerminalTests extends CliToolTestCase {
+import org.elasticsearch.test.ESTestCase;
+
+public class TerminalTests extends ESTestCase {
     public void testVerbosity() throws Exception {
         MockTerminal terminal = new MockTerminal();
         terminal.setVerbosity(Terminal.Verbosity.SILENT);
@@ -48,7 +50,7 @@ public class TerminalTests extends CliToolTestCase {
         logTerminal.println(verbosity, text);
         String output = logTerminal.getOutput();
         assertTrue(output, output.contains(text));
-        logTerminal.resetOutput();
+        logTerminal.reset();
     }
 
     private void assertNotPrinted(MockTerminal logTerminal, Terminal.Verbosity verbosity, String text) throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
index ebba75b10d8..5d3ac18259d 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
@@ -128,8 +128,8 @@ public class TransportClientHeadersTests extends AbstractClientHeadersTestCase {
         CountDownLatch clusterStateLatch = new CountDownLatch(1);
 
         @Inject
-        public InternalTransportService(Settings settings, Transport transport, ThreadPool threadPool, NamedWriteableRegistry namedWriteableRegistry) {
-            super(settings, transport, threadPool, namedWriteableRegistry);
+        public InternalTransportService(Settings settings, Transport transport, ThreadPool threadPool) {
+            super(settings, transport, threadPool);
         }
 
         @Override @SuppressWarnings("unchecked")
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
index ca83deeef1b..5e67ac42d0e 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
@@ -71,7 +71,7 @@ public class TransportClientNodesServiceTests extends ESTestCase {
                     return  new TestResponse();
                 }
             };
-            transportService = new TransportService(Settings.EMPTY, transport, threadPool, new NamedWriteableRegistry());
+            transportService = new TransportService(Settings.EMPTY, transport, threadPool);
             transportService.start();
             transportService.acceptIncomingRequests();
             transportClientNodesService = new TransportClientNodesService(Settings.EMPTY, ClusterName.DEFAULT, transportService, threadPool, Version.CURRENT);
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
index cefd3a6703a..fc43f4154d1 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -37,6 +38,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.equalTo;
 
@@ -220,7 +222,7 @@ public class ClusterChangedEventTests extends ESTestCase {
             final ClusterState newState = nextState(previousState, changeClusterUUID, addedIndices, delIndices, 0);
             final ClusterChangedEvent event = new ClusterChangedEvent("_na_", newState, previousState);
             final List<String> addsFromEvent = event.indicesCreated();
-            final List<String> delsFromEvent = event.indicesDeleted();
+            final List<String> delsFromEvent = event.indicesDeleted().stream().map((s) -> s.getName()).collect(Collectors.toList());
             Collections.sort(addsFromEvent);
             Collections.sort(delsFromEvent);
             assertThat(addsFromEvent, equalTo(addedIndices));
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
index 9a8e8fb7268..28b0b7e18cf 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.cluster;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
-
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionModule;
@@ -35,6 +34,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
index 24635a980a7..4a930bc9c28 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
@@ -35,6 +35,7 @@ import org.elasticsearch.common.inject.ModuleTestCase;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 
@@ -48,19 +49,7 @@ public class ClusterModuleTests extends ModuleTestCase {
 
     static class FakeShardsAllocator implements ShardsAllocator {
         @Override
-        public void applyStartedShards(StartedRerouteAllocation allocation) {}
-        @Override
-        public void applyFailedShards(FailedRerouteAllocation allocation) {}
-        @Override
-        public boolean allocateUnassigned(RoutingAllocation allocation) {
-            return false;
-        }
-        @Override
-        public boolean rebalance(RoutingAllocation allocation) {
-            return false;
-        }
-        @Override
-        public boolean moveShards(RoutingAllocation allocation) {
+        public boolean allocate(RoutingAllocation allocation) {
             return false;
         }
     }
@@ -83,7 +72,7 @@ public class ClusterModuleTests extends ModuleTestCase {
 
     public void testRegisterClusterDynamicSetting() {
         SettingsModule module = new SettingsModule(Settings.EMPTY);
-        module.registerSetting(Setting.boolSetting("foo.bar", false, true, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.boolSetting("foo.bar", false, Property.Dynamic, Property.NodeScope));
         assertInstanceBinding(module, ClusterSettings.class, service -> service.hasDynamicSetting("foo.bar"));
     }
 
@@ -98,8 +87,8 @@ public class ClusterModuleTests extends ModuleTestCase {
 
     public void testRegisterIndexDynamicSetting() {
         SettingsModule module = new SettingsModule(Settings.EMPTY);
-        module.registerSetting(Setting.boolSetting("foo.bar", false, true, Setting.Scope.INDEX));
-        assertInstanceBinding(module, IndexScopedSettings.class, service -> service.hasDynamicSetting("foo.bar"));
+        module.registerSetting(Setting.boolSetting("index.foo.bar", false, Property.Dynamic, Property.IndexScope));
+        assertInstanceBinding(module, IndexScopedSettings.class, service -> service.hasDynamicSetting("index.foo.bar"));
     }
 
     public void testRegisterAllocationDeciderDuplicate() {
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterServiceIT.java
deleted file mode 100644
index f5c99fd5f7e..00000000000
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterServiceIT.java
+++ /dev/null
@@ -1,1351 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.cluster;
-
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.service.InternalClusterService;
-import org.elasticsearch.cluster.service.PendingClusterTask;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.component.LifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.Singleton;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.discovery.zen.ZenDiscovery;
-import org.elasticsearch.node.Node;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.test.InternalTestCluster;
-import org.elasticsearch.test.MockLogAppender;
-import org.elasticsearch.test.junit.annotations.TestLogging;
-import org.elasticsearch.threadpool.ThreadPool;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.BrokenBarrierException;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.elasticsearch.common.settings.Settings.settingsBuilder;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-
-/**
- *
- */
-@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
-@ESIntegTestCase.SuppressLocalMode
-public class ClusterServiceIT extends ESIntegTestCase {
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return pluginList(TestPlugin.class);
-    }
-
-    public void testTimeoutUpdateTask() throws Exception {
-        Settings settings = settingsBuilder()
-                .put("discovery.type", "local")
-                .build();
-        internalCluster().startNode(settings);
-        ClusterService clusterService1 = internalCluster().getInstance(ClusterService.class);
-        final CountDownLatch block = new CountDownLatch(1);
-        clusterService1.submitStateUpdateTask("test1", new ClusterStateUpdateTask() {
-            @Override
-            public ClusterState execute(ClusterState currentState) {
-                try {
-                    block.await();
-                } catch (InterruptedException e) {
-                    throw new RuntimeException(e);
-                }
-                return currentState;
-            }
-
-            @Override
-            public void onFailure(String source, Throwable t) {
-                throw new RuntimeException(t);
-            }
-        });
-
-        final CountDownLatch timedOut = new CountDownLatch(1);
-        final AtomicBoolean executeCalled = new AtomicBoolean();
-        clusterService1.submitStateUpdateTask("test2", new ClusterStateUpdateTask() {
-            @Override
-            public TimeValue timeout() {
-                return TimeValue.timeValueMillis(2);
-            }
-
-            @Override
-            public void onFailure(String source, Throwable t) {
-                timedOut.countDown();
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) {
-                executeCalled.set(true);
-                return currentState;
-            }
-
-            @Override
-            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-            }
-        });
-
-        timedOut.await();
-        block.countDown();
-        final CountDownLatch allProcessed = new CountDownLatch(1);
-        clusterService1.submitStateUpdateTask("test3", new ClusterStateUpdateTask() {
-            @Override
-            public void onFailure(String source, Throwable t) {
-                throw new RuntimeException(t);
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) {
-                allProcessed.countDown();
-                return currentState;
-            }
-
-        });
-        allProcessed.await(); // executed another task to double check that execute on the timed out update task is not called...
-        assertThat(executeCalled.get(), equalTo(false));
-    }
-
-    public void testAckedUpdateTask() throws Exception {
-        Settings settings = settingsBuilder()
-                .put("discovery.type", "local")
-                .build();
-        internalCluster().startNode(settings);
-        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
-
-        final AtomicBoolean allNodesAcked = new AtomicBoolean(false);
-        final AtomicBoolean ackTimeout = new AtomicBoolean(false);
-        final AtomicBoolean onFailure = new AtomicBoolean(false);
-        final AtomicBoolean executed = new AtomicBoolean(false);
-        final CountDownLatch latch = new CountDownLatch(1);
-        final CountDownLatch processedLatch = new CountDownLatch(1);
-        clusterService.submitStateUpdateTask("test", new AckedClusterStateUpdateTask<Void>(null, null) {
-            @Override
-            protected Void newResponse(boolean acknowledged) {
-                return null;
-            }
-
-            @Override
-            public boolean mustAck(DiscoveryNode discoveryNode) {
-                return true;
-            }
-
-            @Override
-            public void onAllNodesAcked(@Nullable Throwable t) {
-                allNodesAcked.set(true);
-                latch.countDown();
-            }
-
-            @Override
-            public void onAckTimeout() {
-                ackTimeout.set(true);
-                latch.countDown();
-            }
-
-            @Override
-            public TimeValue ackTimeout() {
-                return TimeValue.timeValueSeconds(10);
-            }
-
-            @Override
-            public TimeValue timeout() {
-                return TimeValue.timeValueSeconds(10);
-            }
-
-            @Override
-            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                processedLatch.countDown();
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
-                executed.set(true);
-                return ClusterState.builder(currentState).build();
-            }
-
-            @Override
-            public void onFailure(String source, Throwable t) {
-                logger.error("failed to execute callback in test {}", t, source);
-                onFailure.set(true);
-                latch.countDown();
-            }
-        });
-
-        ensureGreen();
-        assertThat(latch.await(1, TimeUnit.SECONDS), equalTo(true));
-
-        assertThat(allNodesAcked.get(), equalTo(true));
-        assertThat(ackTimeout.get(), equalTo(false));
-        assertThat(executed.get(), equalTo(true));
-        assertThat(onFailure.get(), equalTo(false));
-
-        assertThat(processedLatch.await(1, TimeUnit.SECONDS), equalTo(true));
-    }
-
-    public void testAckedUpdateTaskSameClusterState() throws Exception {
-        Settings settings = settingsBuilder()
-                .put("discovery.type", "local")
-                .build();
-        internalCluster().startNode(settings);
-        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
-
-        final AtomicBoolean allNodesAcked = new AtomicBoolean(false);
-        final AtomicBoolean ackTimeout = new AtomicBoolean(false);
-        final AtomicBoolean onFailure = new AtomicBoolean(false);
-        final AtomicBoolean executed = new AtomicBoolean(false);
-        final CountDownLatch latch = new CountDownLatch(1);
-        final CountDownLatch processedLatch = new CountDownLatch(1);
-        clusterService.submitStateUpdateTask("test", new AckedClusterStateUpdateTask<Void>(null, null) {
-            @Override
-            protected Void newResponse(boolean acknowledged) {
-                return null;
-            }
-
-            @Override
-            public void onAllNodesAcked(@Nullable Throwable t) {
-                allNodesAcked.set(true);
-                latch.countDown();
-            }
-
-            @Override
-            public void onAckTimeout() {
-                ackTimeout.set(true);
-                latch.countDown();
-            }
-
-            @Override
-            public TimeValue ackTimeout() {
-                return TimeValue.timeValueSeconds(10);
-            }
-
-            @Override
-            public TimeValue timeout() {
-                return TimeValue.timeValueSeconds(10);
-            }
-
-            @Override
-            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                processedLatch.countDown();
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
-                executed.set(true);
-                return currentState;
-            }
-
-            @Override
-            public void onFailure(String source, Throwable t) {
-                logger.error("failed to execute callback in test {}", t, source);
-                onFailure.set(true);
-                latch.countDown();
-            }
-        });
-
-        ensureGreen();
-        assertThat(latch.await(1, TimeUnit.SECONDS), equalTo(true));
-
-        assertThat(allNodesAcked.get(), equalTo(true));
-        assertThat(ackTimeout.get(), equalTo(false));
-        assertThat(executed.get(), equalTo(true));
-        assertThat(onFailure.get(), equalTo(false));
-
-        assertThat(processedLatch.await(1, TimeUnit.SECONDS), equalTo(true));
-    }
-
-    public void testMasterAwareExecution() throws Exception {
-        Settings settings = settingsBuilder()
-                .put("discovery.type", "local")
-                .build();
-
-        InternalTestCluster.Async<String> master = internalCluster().startNodeAsync(settings);
-        InternalTestCluster.Async<String> nonMaster = internalCluster().startNodeAsync(settingsBuilder().put(settings).put(Node.NODE_MASTER_SETTING.getKey(), false).build());
-        master.get();
-        ensureGreen(); // make sure we have a cluster
-
-        ClusterService clusterService = internalCluster().getInstance(ClusterService.class, nonMaster.get());
-
-        final boolean[] taskFailed = {false};
-        final CountDownLatch latch1 = new CountDownLatch(1);
-        clusterService.submitStateUpdateTask("test", new ClusterStateUpdateTask() {
-            @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
-                latch1.countDown();
-                return currentState;
-            }
-
-            @Override
-            public void onFailure(String source, Throwable t) {
-                taskFailed[0] = true;
-                latch1.countDown();
-            }
-        });
-
-        latch1.await();
-        assertTrue("cluster state update task was executed on a non-master", taskFailed[0]);
-
-        taskFailed[0] = true;
-        final CountDownLatch latch2 = new CountDownLatch(1);
-        clusterService.submitStateUpdateTask("test", new ClusterStateUpdateTask() {
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
-                taskFailed[0] = false;
-                latch2.countDown();
-                return currentState;
-            }
-
-            @Override
-            public void onFailure(String source, Throwable t) {
-                taskFailed[0] = true;
-                latch2.countDown();
-            }
-        });
-        latch2.await();
-        assertFalse("non-master cluster state update task was not executed", taskFailed[0]);
-    }
-
-    public void testAckedUpdateTaskNoAckExpected() throws Exception {
-        Settings settings = settingsBuilder()
-                .put("discovery.type", "local")
-                .build();
-        internalCluster().startNode(settings);
-        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
-
-        final AtomicBoolean allNodesAcked = new AtomicBoolean(false);
-        final AtomicBoolean ackTimeout = new AtomicBoolean(false);
-        final AtomicBoolean onFailure = new AtomicBoolean(false);
-        final AtomicBoolean executed = new AtomicBoolean(false);
-        final CountDownLatch latch = new CountDownLatch(1);
-        clusterService.submitStateUpdateTask("test", new AckedClusterStateUpdateTask<Void>(null, null) {
-            @Override
-            protected Void newResponse(boolean acknowledged) {
-                return null;
-            }
-
-            @Override
-            public boolean mustAck(DiscoveryNode discoveryNode) {
-                return false;
-            }
-
-            @Override
-            public void onAllNodesAcked(@Nullable Throwable t) {
-                allNodesAcked.set(true);
-                latch.countDown();
-            }
-
-            @Override
-            public void onAckTimeout() {
-                ackTimeout.set(true);
-                latch.countDown();
-            }
-
-            @Override
-            public TimeValue ackTimeout() {
-                return TimeValue.timeValueSeconds(10);
-            }
-
-            @Override
-            public TimeValue timeout() {
-                return TimeValue.timeValueSeconds(10);
-            }
-
-            @Override
-            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
-                executed.set(true);
-                return ClusterState.builder(currentState).build();
-            }
-
-            @Override
-            public void onFailure(String source, Throwable t) {
-                logger.error("failed to execute callback in test {}", t, source);
-                onFailure.set(true);
-                latch.countDown();
-            }
-        });
-
-        ensureGreen();
-        assertThat(latch.await(1, TimeUnit.SECONDS), equalTo(true));
-
-        assertThat(allNodesAcked.get(), equalTo(true));
-        assertThat(ackTimeout.get(), equalTo(false));
-        assertThat(executed.get(), equalTo(true));
-        assertThat(onFailure.get(), equalTo(false));
-    }
-
-    public void testAckedUpdateTaskTimeoutZero() throws Exception {
-        Settings settings = settingsBuilder()
-                .put("discovery.type", "local")
-                .build();
-        internalCluster().startNode(settings);
-        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
-
-        final AtomicBoolean allNodesAcked = new AtomicBoolean(false);
-        final AtomicBoolean ackTimeout = new AtomicBoolean(false);
-        final AtomicBoolean onFailure = new AtomicBoolean(false);
-        final AtomicBoolean executed = new AtomicBoolean(false);
-        final CountDownLatch latch = new CountDownLatch(1);
-        final CountDownLatch processedLatch = new CountDownLatch(1);
-        clusterService.submitStateUpdateTask("test", new AckedClusterStateUpdateTask<Void>(null, null) {
-            @Override
-            protected Void newResponse(boolean acknowledged) {
-                return null;
-            }
-
-            @Override
-            public boolean mustAck(DiscoveryNode discoveryNode) {
-                return false;
-            }
-
-            @Override
-            public void onAllNodesAcked(@Nullable Throwable t) {
-                allNodesAcked.set(true);
-                latch.countDown();
-            }
-
-            @Override
-            public void onAckTimeout() {
-                ackTimeout.set(true);
-                latch.countDown();
-            }
-
-            @Override
-            public TimeValue ackTimeout() {
-                return TimeValue.timeValueSeconds(0);
-            }
-
-            @Override
-            public TimeValue timeout() {
-                return TimeValue.timeValueSeconds(10);
-            }
-
-            @Override
-            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                processedLatch.countDown();
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
-                executed.set(true);
-                return ClusterState.builder(currentState).build();
-            }
-
-            @Override
-            public void onFailure(String source, Throwable t) {
-                logger.error("failed to execute callback in test {}", t, source);
-                onFailure.set(true);
-                latch.countDown();
-            }
-        });
-
-        ensureGreen();
-        assertThat(latch.await(1, TimeUnit.SECONDS), equalTo(true));
-
-        assertThat(allNodesAcked.get(), equalTo(false));
-        assertThat(ackTimeout.get(), equalTo(true));
-        assertThat(executed.get(), equalTo(true));
-        assertThat(onFailure.get(), equalTo(false));
-
-        assertThat(processedLatch.await(1, TimeUnit.SECONDS), equalTo(true));
-    }
-
-    @TestLogging("_root:debug,action.admin.cluster.tasks:trace")
-    public void testPendingUpdateTask() throws Exception {
-        Settings settings = settingsBuilder()
-                .put("discovery.type", "local")
-                .build();
-        String node_0 = internalCluster().startNode(settings);
-        internalCluster().startNodeClient(settings);
-
-        final ClusterService clusterService = internalCluster().getInstance(ClusterService.class, node_0);
-        final CountDownLatch block1 = new CountDownLatch(1);
-        final CountDownLatch invoked1 = new CountDownLatch(1);
-        clusterService.submitStateUpdateTask("1", new ClusterStateUpdateTask() {
-            @Override
-            public ClusterState execute(ClusterState currentState) {
-                invoked1.countDown();
-                try {
-                    block1.await();
-                } catch (InterruptedException e) {
-                    fail();
-                }
-                return currentState;
-            }
-
-            @Override
-            public void onFailure(String source, Throwable t) {
-                invoked1.countDown();
-                fail();
-            }
-        });
-        invoked1.await();
-        final CountDownLatch invoked2 = new CountDownLatch(9);
-        for (int i = 2; i <= 10; i++) {
-            clusterService.submitStateUpdateTask(Integer.toString(i), new ClusterStateUpdateTask() {
-                @Override
-                public ClusterState execute(ClusterState currentState) {
-                    return currentState;
-                }
-
-                @Override
-                public void onFailure(String source, Throwable t) {
-                    fail();
-                }
-
-                @Override
-                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                    invoked2.countDown();
-                }
-            });
-        }
-
-        // there might be other tasks in this node, make sure to only take the ones we add into account in this test
-
-        // The tasks can be re-ordered, so we need to check out-of-order
-        Set<String> controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9", "10"));
-        List<PendingClusterTask> pendingClusterTasks = clusterService.pendingTasks();
-        assertThat(pendingClusterTasks.size(), greaterThanOrEqualTo(10));
-        assertThat(pendingClusterTasks.get(0).getSource().string(), equalTo("1"));
-        assertThat(pendingClusterTasks.get(0).isExecuting(), equalTo(true));
-        for (PendingClusterTask task : pendingClusterTasks) {
-            controlSources.remove(task.getSource().string());
-        }
-        assertTrue(controlSources.isEmpty());
-
-        controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9", "10"));
-        PendingClusterTasksResponse response = internalCluster().clientNodeClient().admin().cluster().preparePendingClusterTasks().execute().actionGet();
-        assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(10));
-        assertThat(response.pendingTasks().get(0).getSource().string(), equalTo("1"));
-        assertThat(response.pendingTasks().get(0).isExecuting(), equalTo(true));
-        for (PendingClusterTask task : response) {
-            controlSources.remove(task.getSource().string());
-        }
-        assertTrue(controlSources.isEmpty());
-        block1.countDown();
-        invoked2.await();
-
-        // whenever we test for no tasks, we need to awaitBusy since this is a live node
-        assertTrue(awaitBusy(() -> clusterService.pendingTasks().isEmpty()));
-        waitNoPendingTasksOnAll();
-
-        final CountDownLatch block2 = new CountDownLatch(1);
-        final CountDownLatch invoked3 = new CountDownLatch(1);
-        clusterService.submitStateUpdateTask("1", new ClusterStateUpdateTask() {
-            @Override
-            public ClusterState execute(ClusterState currentState) {
-                invoked3.countDown();
-                try {
-                    block2.await();
-                } catch (InterruptedException e) {
-                    fail();
-                }
-                return currentState;
-            }
-
-            @Override
-            public void onFailure(String source, Throwable t) {
-                invoked3.countDown();
-                fail();
-            }
-        });
-        invoked3.await();
-
-        for (int i = 2; i <= 5; i++) {
-            clusterService.submitStateUpdateTask(Integer.toString(i), new ClusterStateUpdateTask() {
-                @Override
-                public ClusterState execute(ClusterState currentState) {
-                    return currentState;
-                }
-
-                @Override
-                public void onFailure(String source, Throwable t) {
-                    fail();
-                }
-            });
-        }
-        Thread.sleep(100);
-
-        pendingClusterTasks = clusterService.pendingTasks();
-        assertThat(pendingClusterTasks.size(), greaterThanOrEqualTo(5));
-        controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5"));
-        for (PendingClusterTask task : pendingClusterTasks) {
-            controlSources.remove(task.getSource().string());
-        }
-        assertTrue(controlSources.isEmpty());
-
-        response = internalCluster().clientNodeClient().admin().cluster().preparePendingClusterTasks().get();
-        assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(5));
-        controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5"));
-        for (PendingClusterTask task : response) {
-            if (controlSources.remove(task.getSource().string())) {
-                assertThat(task.getTimeInQueueInMillis(), greaterThan(0L));
-            }
-        }
-        assertTrue(controlSources.isEmpty());
-        block2.countDown();
-    }
-
-    public void testLocalNodeMasterListenerCallbacks() throws Exception {
-        Settings settings = settingsBuilder()
-                .put("discovery.type", "zen")
-                .put("discovery.zen.minimum_master_nodes", 1)
-                .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "400ms")
-                .put("discovery.initial_state_timeout", "500ms")
-                .build();
-
-        String node_0 = internalCluster().startNode(settings);
-        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
-        MasterAwareService testService = internalCluster().getInstance(MasterAwareService.class);
-
-        ClusterHealthResponse clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("1").get();
-        assertThat(clusterHealth.isTimedOut(), equalTo(false));
-
-        // the first node should be a master as the minimum required is 1
-        assertThat(clusterService.state().nodes().masterNode(), notNullValue());
-        assertThat(clusterService.state().nodes().localNodeMaster(), is(true));
-        assertThat(testService.master(), is(true));
-
-        String node_1 = internalCluster().startNode(settings);
-        final ClusterService clusterService1 = internalCluster().getInstance(ClusterService.class, node_1);
-        MasterAwareService testService1 = internalCluster().getInstance(MasterAwareService.class, node_1);
-
-        clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("2").get();
-        assertThat(clusterHealth.isTimedOut(), equalTo(false));
-
-        // the second node should not be the master as node1 is already the master.
-        assertThat(clusterService1.state().nodes().localNodeMaster(), is(false));
-        assertThat(testService1.master(), is(false));
-
-        internalCluster().stopCurrentMasterNode();
-        clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("1").get();
-        assertThat(clusterHealth.isTimedOut(), equalTo(false));
-
-        // now that node0 is closed, node1 should be elected as master
-        assertThat(clusterService1.state().nodes().localNodeMaster(), is(true));
-        assertThat(testService1.master(), is(true));
-
-        // start another node and set min_master_node
-        internalCluster().startNode(Settings.builder().put(settings));
-        assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());
-
-        Settings transientSettings = settingsBuilder()
-                .put("discovery.zen.minimum_master_nodes", 2)
-                .build();
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(transientSettings).get();
-
-        // and shutdown the second node
-        internalCluster().stopRandomNonMasterNode();
-
-        // there should not be any master as the minimum number of required eligible masters is not met
-        awaitBusy(() -> clusterService1.state().nodes().masterNode() == null && clusterService1.state().status() == ClusterState.ClusterStateStatus.APPLIED);
-        assertThat(testService1.master(), is(false));
-
-        // bring the node back up
-        String node_2 = internalCluster().startNode(Settings.builder().put(settings).put(transientSettings));
-        ClusterService clusterService2 = internalCluster().getInstance(ClusterService.class, node_2);
-        MasterAwareService testService2 = internalCluster().getInstance(MasterAwareService.class, node_2);
-
-        // make sure both nodes see each other otherwise the masternode below could be null if node 2 is master and node 1 did'r receive the updated cluster state...
-        assertThat(internalCluster().client(node_1).admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setLocal(true).setWaitForNodes("2").get().isTimedOut(), is(false));
-        assertThat(internalCluster().client(node_2).admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setLocal(true).setWaitForNodes("2").get().isTimedOut(), is(false));
-
-        // now that we started node1 again, a new master should be elected
-        assertThat(clusterService2.state().nodes().masterNode(), is(notNullValue()));
-        if (node_2.equals(clusterService2.state().nodes().masterNode().name())) {
-            assertThat(testService1.master(), is(false));
-            assertThat(testService2.master(), is(true));
-        } else {
-            assertThat(testService1.master(), is(true));
-            assertThat(testService2.master(), is(false));
-        }
-    }
-
-    /**
-     * Note, this test can only work as long as we have a single thread executor executing the state update tasks!
-     */
-    public void testPrioritizedTasks() throws Exception {
-        Settings settings = settingsBuilder()
-                .put("discovery.type", "local")
-                .build();
-        internalCluster().startNode(settings);
-        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
-        BlockingTask block = new BlockingTask(Priority.IMMEDIATE);
-        clusterService.submitStateUpdateTask("test", block);
-        int taskCount = randomIntBetween(5, 20);
-        Priority[] priorities = Priority.values();
-
-        // will hold all the tasks in the order in which they were executed
-        List<PrioritizedTask> tasks = new ArrayList<>(taskCount);
-        CountDownLatch latch = new CountDownLatch(taskCount);
-        for (int i = 0; i < taskCount; i++) {
-            Priority priority = priorities[randomIntBetween(0, priorities.length - 1)];
-            clusterService.submitStateUpdateTask("test", new PrioritizedTask(priority, latch, tasks));
-        }
-
-        block.release();
-        latch.await();
-
-        Priority prevPriority = null;
-        for (PrioritizedTask task : tasks) {
-            if (prevPriority == null) {
-                prevPriority = task.priority();
-            } else {
-                assertThat(task.priority().sameOrAfter(prevPriority), is(true));
-            }
-        }
-    }
-
-    /*
-     * test that a listener throwing an exception while handling a
-     * notification does not prevent publication notification to the
-     * executor
-     */
-    public void testClusterStateTaskListenerThrowingExceptionIsOkay() throws InterruptedException {
-        Settings settings = settingsBuilder()
-            .put("discovery.type", "local")
-            .build();
-        internalCluster().startNode(settings);
-        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
-
-        final CountDownLatch latch = new CountDownLatch(1);
-        AtomicBoolean published = new AtomicBoolean();
-
-        clusterService.submitStateUpdateTask(
-            "testClusterStateTaskListenerThrowingExceptionIsOkay",
-            new Object(),
-            ClusterStateTaskConfig.build(Priority.NORMAL),
-            new ClusterStateTaskExecutor<Object>() {
-                @Override
-                public boolean runOnlyOnMaster() {
-                    return false;
-                }
-
-                @Override
-                public BatchResult<Object> execute(ClusterState currentState, List<Object> tasks) throws Exception {
-                    ClusterState newClusterState = ClusterState.builder(currentState).build();
-                    return BatchResult.builder().successes(tasks).build(newClusterState);
-                }
-
-                @Override
-                public void clusterStatePublished(ClusterState newClusterState) {
-                    published.set(true);
-                    latch.countDown();
-                }
-            },
-            new ClusterStateTaskListener() {
-                @Override
-                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                    throw new IllegalStateException(source);
-                }
-
-                @Override
-                public void onFailure(String source, Throwable t) {
-                }
-            }
-        );
-
-        latch.await();
-        assertTrue(published.get());
-    }
-
-    // test that for a single thread, tasks are executed in the order
-    // that they are submitted
-    public void testClusterStateUpdateTasksAreExecutedInOrder() throws BrokenBarrierException, InterruptedException {
-        Settings settings = settingsBuilder()
-            .put("discovery.type", "local")
-            .build();
-        internalCluster().startNode(settings);
-        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
-
-        class TaskExecutor implements ClusterStateTaskExecutor<Integer> {
-            List<Integer> tasks = new ArrayList<>();
-
-            @Override
-            public BatchResult<Integer> execute(ClusterState currentState, List<Integer> tasks) throws Exception {
-                this.tasks.addAll(tasks);
-                return BatchResult.<Integer>builder().successes(tasks).build(ClusterState.builder(currentState).build());
-            }
-
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
-        }
-
-        int numberOfThreads = randomIntBetween(2, 8);
-        TaskExecutor[] executors = new TaskExecutor[numberOfThreads];
-        for (int i = 0; i < numberOfThreads; i++) {
-            executors[i] = new TaskExecutor();
-        }
-
-        int tasksSubmittedPerThread = randomIntBetween(2, 1024);
-
-        CopyOnWriteArrayList<Tuple<String, Throwable>> failures = new CopyOnWriteArrayList<>();
-        CountDownLatch updateLatch = new CountDownLatch(numberOfThreads * tasksSubmittedPerThread);
-
-        ClusterStateTaskListener listener = new ClusterStateTaskListener() {
-            @Override
-            public void onFailure(String source, Throwable t) {
-                logger.error("unexpected failure: [{}]", t, source);
-                failures.add(new Tuple<>(source, t));
-                updateLatch.countDown();
-            }
-
-            @Override
-            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                updateLatch.countDown();
-            }
-        };
-
-        CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads);
-
-        for (int i = 0; i < numberOfThreads; i++) {
-            final int index = i;
-            Thread thread = new Thread(() -> {
-                try {
-                    barrier.await();
-                    for (int j = 0; j < tasksSubmittedPerThread; j++) {
-                        clusterService.submitStateUpdateTask("[" + index + "][" + j + "]", j, ClusterStateTaskConfig.build(randomFrom(Priority.values())), executors[index], listener);
-                    }
-                    barrier.await();
-                } catch (InterruptedException | BrokenBarrierException e) {
-                    throw new AssertionError(e);
-                }
-            });
-            thread.start();
-        }
-
-        // wait for all threads to be ready
-        barrier.await();
-        // wait for all threads to finish
-        barrier.await();
-
-        updateLatch.await();
-
-        assertThat(failures, empty());
-
-        for (int i = 0; i < numberOfThreads; i++) {
-            assertEquals(tasksSubmittedPerThread, executors[i].tasks.size());
-            for (int j = 0; j < tasksSubmittedPerThread; j++) {
-                assertNotNull(executors[i].tasks.get(j));
-                assertEquals("cluster state update task executed out of order", j, (int)executors[i].tasks.get(j));
-            }
-        }
-    }
-
-    public void testClusterStateBatchedUpdates() throws BrokenBarrierException, InterruptedException {
-        Settings settings = settingsBuilder()
-                .put("discovery.type", "local")
-                .build();
-        internalCluster().startNode(settings);
-        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
-
-        AtomicInteger counter = new AtomicInteger();
-        class Task {
-            private AtomicBoolean state = new AtomicBoolean();
-
-            public void execute() {
-                if (!state.compareAndSet(false, true)) {
-                    throw new IllegalStateException();
-                } else {
-                    counter.incrementAndGet();
-                }
-            }
-        }
-
-        int numberOfThreads = randomIntBetween(2, 8);
-        int tasksSubmittedPerThread = randomIntBetween(1, 1024);
-        int numberOfExecutors = Math.max(1, numberOfThreads / 4);
-        final Semaphore semaphore = new Semaphore(numberOfExecutors);
-
-        class TaskExecutor implements ClusterStateTaskExecutor<Task> {
-            private AtomicInteger counter = new AtomicInteger();
-            private AtomicInteger batches = new AtomicInteger();
-            private AtomicInteger published = new AtomicInteger();
-
-            @Override
-            public BatchResult<Task> execute(ClusterState currentState, List<Task> tasks) throws Exception {
-                tasks.forEach(task -> task.execute());
-                counter.addAndGet(tasks.size());
-                ClusterState maybeUpdatedClusterState = currentState;
-                if (randomBoolean()) {
-                    maybeUpdatedClusterState = ClusterState.builder(currentState).build();
-                    batches.incrementAndGet();
-                    semaphore.acquire();
-                }
-                return BatchResult.<Task>builder().successes(tasks).build(maybeUpdatedClusterState);
-            }
-
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
-
-            @Override
-            public void clusterStatePublished(ClusterState newClusterState) {
-                published.incrementAndGet();
-                semaphore.release();
-            }
-        }
-
-        ConcurrentMap<String, AtomicInteger> counters = new ConcurrentHashMap<>();
-        CountDownLatch updateLatch = new CountDownLatch(numberOfThreads * tasksSubmittedPerThread);
-        ClusterStateTaskListener listener = new ClusterStateTaskListener() {
-            @Override
-            public void onFailure(String source, Throwable t) {
-                assert false;
-            }
-
-            @Override
-            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                counters.computeIfAbsent(source, key -> new AtomicInteger()).incrementAndGet();
-                updateLatch.countDown();
-            }
-        };
-
-        List<TaskExecutor> executors = new ArrayList<>();
-        for (int i = 0; i < numberOfExecutors; i++) {
-            executors.add(new TaskExecutor());
-        }
-
-        // randomly assign tasks to executors
-        List<TaskExecutor> assignments = new ArrayList<>();
-        for (int i = 0; i < numberOfThreads; i++) {
-            for (int j = 0; j < tasksSubmittedPerThread; j++) {
-                assignments.add(randomFrom(executors));
-            }
-        }
-
-        Map<TaskExecutor, Integer> counts = new HashMap<>();
-        for (TaskExecutor executor : assignments) {
-            counts.merge(executor, 1, (previous, one) -> previous + one);
-        }
-
-        CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads);
-        for (int i = 0; i < numberOfThreads; i++) {
-            final int index = i;
-            Thread thread = new Thread(() -> {
-                try {
-                    barrier.await();
-                    for (int j = 0; j < tasksSubmittedPerThread; j++) {
-                        ClusterStateTaskExecutor<Task> executor = assignments.get(index * tasksSubmittedPerThread + j);
-                        clusterService.submitStateUpdateTask(
-                                Thread.currentThread().getName(),
-                                new Task(),
-                                ClusterStateTaskConfig.build(randomFrom(Priority.values())),
-                                executor,
-                                listener);
-                    }
-                    barrier.await();
-                } catch (BrokenBarrierException | InterruptedException e) {
-                    throw new AssertionError(e);
-                }
-            });
-            thread.start();
-        }
-
-        // wait for all threads to be ready
-        barrier.await();
-        // wait for all threads to finish
-        barrier.await();
-
-        // wait until all the cluster state updates have been processed
-        updateLatch.await();
-        // and until all of the publication callbacks have completed
-        semaphore.acquire(numberOfExecutors);
-
-        // assert the number of executed tasks is correct
-        assertEquals(numberOfThreads * tasksSubmittedPerThread, counter.get());
-
-        // assert each executor executed the correct number of tasks
-        for (TaskExecutor executor : executors) {
-            if (counts.containsKey(executor)) {
-                assertEquals((int) counts.get(executor), executor.counter.get());
-                assertEquals(executor.batches.get(), executor.published.get());
-            }
-        }
-
-        // assert the correct number of clusterStateProcessed events were triggered
-        for (Map.Entry<String, AtomicInteger> entry : counters.entrySet()) {
-            assertEquals(entry.getValue().get(), tasksSubmittedPerThread);
-        }
-    }
-
-    @TestLogging("cluster:TRACE") // To ensure that we log cluster state events on TRACE level
-    public void testClusterStateUpdateLogging() throws Exception {
-        Settings settings = settingsBuilder()
-                .put("discovery.type", "local")
-                .build();
-        internalCluster().startNode(settings);
-        ClusterService clusterService1 = internalCluster().getInstance(ClusterService.class);
-        MockLogAppender mockAppender = new MockLogAppender();
-        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test1", "cluster.service", Level.DEBUG, "*processing [test1]: took * no change in cluster_state"));
-        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test2", "cluster.service", Level.TRACE, "*failed to execute cluster state update in *"));
-        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test3", "cluster.service", Level.DEBUG, "*processing [test3]: took * done applying updated cluster_state (version: *, uuid: *)"));
-
-        Logger rootLogger = Logger.getRootLogger();
-        rootLogger.addAppender(mockAppender);
-        try {
-            final CountDownLatch latch = new CountDownLatch(4);
-            clusterService1.submitStateUpdateTask("test1", new ClusterStateUpdateTask() {
-                @Override
-                public ClusterState execute(ClusterState currentState) throws Exception {
-                    return currentState;
-                }
-
-                @Override
-                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                    latch.countDown();
-                }
-
-                @Override
-                public void onFailure(String source, Throwable t) {
-                    fail();
-                }
-            });
-            clusterService1.submitStateUpdateTask("test2", new ClusterStateUpdateTask() {
-                @Override
-                public ClusterState execute(ClusterState currentState) {
-                    throw new IllegalArgumentException("Testing handling of exceptions in the cluster state task");
-                }
-
-                @Override
-                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                    fail();
-                }
-
-                @Override
-                public void onFailure(String source, Throwable t) {
-                    latch.countDown();
-                }
-            });
-            clusterService1.submitStateUpdateTask("test3", new ClusterStateUpdateTask() {
-                @Override
-                public ClusterState execute(ClusterState currentState) {
-                    return ClusterState.builder(currentState).incrementVersion().build();
-                }
-
-                @Override
-                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                    latch.countDown();
-                }
-
-                @Override
-                public void onFailure(String source, Throwable t) {
-                    fail();
-                }
-            });
-            // Additional update task to make sure all previous logging made it to the logger
-            // We don't check logging for this on since there is no guarantee that it will occur before our check
-            clusterService1.submitStateUpdateTask("test4", new ClusterStateUpdateTask() {
-                @Override
-                public ClusterState execute(ClusterState currentState) {
-                    return currentState;
-                }
-
-                @Override
-                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                    latch.countDown();
-                }
-
-                @Override
-                public void onFailure(String source, Throwable t) {
-                    fail();
-                }
-            });
-            assertThat(latch.await(1, TimeUnit.SECONDS), equalTo(true));
-        } finally {
-            rootLogger.removeAppender(mockAppender);
-        }
-        mockAppender.assertAllExpectationsMatched();
-    }
-
-    @TestLogging("cluster:WARN") // To ensure that we log cluster state events on WARN level
-    public void testLongClusterStateUpdateLogging() throws Exception {
-        Settings settings = settingsBuilder()
-                .put("discovery.type", "local")
-                .put(InternalClusterService.CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.getKey(), "10s")
-                .build();
-        internalCluster().startNode(settings);
-        ClusterService clusterService1 = internalCluster().getInstance(ClusterService.class);
-        MockLogAppender mockAppender = new MockLogAppender();
-        mockAppender.addExpectation(new MockLogAppender.UnseenEventExpectation("test1 shouldn't see because setting is too low", "cluster.service", Level.WARN, "*cluster state update task [test1] took * above the warn threshold of *"));
-        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test2", "cluster.service", Level.WARN, "*cluster state update task [test2] took * above the warn threshold of 10ms"));
-        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test3", "cluster.service", Level.WARN, "*cluster state update task [test3] took * above the warn threshold of 10ms"));
-        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test4", "cluster.service", Level.WARN, "*cluster state update task [test4] took * above the warn threshold of 10ms"));
-
-        Logger rootLogger = Logger.getRootLogger();
-        rootLogger.addAppender(mockAppender);
-        try {
-            final CountDownLatch latch = new CountDownLatch(5);
-            final CountDownLatch processedFirstTask = new CountDownLatch(1);
-            clusterService1.submitStateUpdateTask("test1", new ClusterStateUpdateTask() {
-                @Override
-                public ClusterState execute(ClusterState currentState) throws Exception {
-                    Thread.sleep(100);
-                    return currentState;
-                }
-
-                @Override
-                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                    latch.countDown();
-                    processedFirstTask.countDown();
-                }
-
-                @Override
-                public void onFailure(String source, Throwable t) {
-                    fail();
-                }
-            });
-
-            processedFirstTask.await(1, TimeUnit.SECONDS);
-            assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settingsBuilder()
-                    .put(InternalClusterService.CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.getKey(), "10ms")));
-
-            clusterService1.submitStateUpdateTask("test2", new ClusterStateUpdateTask() {
-                @Override
-                public ClusterState execute(ClusterState currentState) throws Exception {
-                    Thread.sleep(100);
-                    throw new IllegalArgumentException("Testing handling of exceptions in the cluster state task");
-                }
-
-                @Override
-                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                    fail();
-                }
-
-                @Override
-                public void onFailure(String source, Throwable t) {
-                    latch.countDown();
-                }
-            });
-            clusterService1.submitStateUpdateTask("test3", new ClusterStateUpdateTask() {
-                @Override
-                public ClusterState execute(ClusterState currentState) throws Exception {
-                    Thread.sleep(100);
-                    return ClusterState.builder(currentState).incrementVersion().build();
-                }
-
-                @Override
-                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                    latch.countDown();
-                }
-
-                @Override
-                public void onFailure(String source, Throwable t) {
-                    fail();
-                }
-            });
-            clusterService1.submitStateUpdateTask("test4", new ClusterStateUpdateTask() {
-                @Override
-                public ClusterState execute(ClusterState currentState) throws Exception {
-                    Thread.sleep(100);
-                    return currentState;
-                }
-
-                @Override
-                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                    latch.countDown();
-                }
-
-                @Override
-                public void onFailure(String source, Throwable t) {
-                    fail();
-                }
-            });
-            // Additional update task to make sure all previous logging made it to the logger
-            // We don't check logging for this on since there is no guarantee that it will occur before our check
-            clusterService1.submitStateUpdateTask("test5", new ClusterStateUpdateTask() {
-                @Override
-                public ClusterState execute(ClusterState currentState) {
-                    return currentState;
-                }
-
-                @Override
-                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                    latch.countDown();
-                }
-
-                @Override
-                public void onFailure(String source, Throwable t) {
-                    fail();
-                }
-            });
-            assertThat(latch.await(5, TimeUnit.SECONDS), equalTo(true));
-        } finally {
-            rootLogger.removeAppender(mockAppender);
-        }
-        mockAppender.assertAllExpectationsMatched();
-    }
-
-    private static class BlockingTask extends ClusterStateUpdateTask {
-        private final CountDownLatch latch = new CountDownLatch(1);
-
-        public BlockingTask(Priority priority) {
-            super(priority);
-        }
-
-        @Override
-        public ClusterState execute(ClusterState currentState) throws Exception {
-            latch.await();
-            return currentState;
-        }
-
-        @Override
-        public void onFailure(String source, Throwable t) {
-        }
-
-        public void release() {
-            latch.countDown();
-        }
-
-    }
-
-    private static class PrioritizedTask extends ClusterStateUpdateTask {
-
-        private final CountDownLatch latch;
-        private final List<PrioritizedTask> tasks;
-
-        private PrioritizedTask(Priority priority, CountDownLatch latch, List<PrioritizedTask> tasks) {
-            super(priority);
-            this.latch = latch;
-            this.tasks = tasks;
-        }
-
-        @Override
-        public ClusterState execute(ClusterState currentState) throws Exception {
-            tasks.add(this);
-            latch.countDown();
-            return currentState;
-        }
-
-        @Override
-        public void onFailure(String source, Throwable t) {
-            latch.countDown();
-        }
-    }
-
-    public static class TestPlugin extends Plugin {
-
-        @Override
-        public String name() {
-            return "test plugin";
-        }
-
-        @Override
-        public String description() {
-            return "test plugin";
-        }
-
-        @Override
-        public Collection<Class<? extends LifecycleComponent>> nodeServices() {
-            List<Class<? extends LifecycleComponent>> services = new ArrayList<>(1);
-            services.add(MasterAwareService.class);
-            return services;
-        }
-    }
-
-    @Singleton
-    public static class MasterAwareService extends AbstractLifecycleComponent<MasterAwareService> implements LocalNodeMasterListener {
-
-        private final ClusterService clusterService;
-        private volatile boolean master;
-
-        @Inject
-        public MasterAwareService(Settings settings, ClusterService clusterService) {
-            super(settings);
-            clusterService.add(this);
-            this.clusterService = clusterService;
-            logger.info("initialized test service");
-        }
-
-        @Override
-        public void onMaster() {
-            logger.info("on master [" + clusterService.localNode() + "]");
-            master = true;
-        }
-
-        @Override
-        public void offMaster() {
-            logger.info("off master [" + clusterService.localNode() + "]");
-            master = false;
-        }
-
-        public boolean master() {
-            return master;
-        }
-
-        @Override
-        protected void doStart() {
-        }
-
-        @Override
-        protected void doStop() {
-        }
-
-        @Override
-        protected void doClose() {
-        }
-
-        @Override
-        public String executorName() {
-            return ThreadPool.Names.SAME;
-        }
-
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
index addb753ff4a..c158fb1c360 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
@@ -639,6 +639,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
                         return new SnapshotsInProgress(new SnapshotsInProgress.Entry(
                                 new SnapshotId(randomName("repo"), randomName("snap")),
                                 randomBoolean(),
+                                randomBoolean(),
                                 SnapshotsInProgress.State.fromValue((byte) randomIntBetween(0, 6)),
                                 Collections.<String>emptyList(),
                                 Math.abs(randomLong()),
diff --git a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
index a74102f6969..424565f13bf 100644
--- a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
@@ -92,22 +92,22 @@ public class DiskUsageTests extends ESTestCase {
     }
 
     public void testFillShardLevelInfo() {
-        final Index index = new Index("test", "_na_");
+        final Index index = new Index("test", "0xdeadbeef");
         ShardRouting test_0 = ShardRouting.newUnassigned(index, 0, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         ShardRoutingHelper.initialize(test_0, "node1");
         ShardRoutingHelper.moveToStarted(test_0);
-        Path test0Path = createTempDir().resolve("indices").resolve("test").resolve("0");
+        Path test0Path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("0");
         CommonStats commonStats0 = new CommonStats();
         commonStats0.store = new StoreStats(100, 1);
         ShardRouting test_1 = ShardRouting.newUnassigned(index, 1, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         ShardRoutingHelper.initialize(test_1, "node2");
         ShardRoutingHelper.moveToStarted(test_1);
-        Path test1Path = createTempDir().resolve("indices").resolve("test").resolve("1");
+        Path test1Path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("1");
         CommonStats commonStats1 = new CommonStats();
         commonStats1.store = new StoreStats(1000, 1);
         ShardStats[] stats  = new ShardStats[] {
-                new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, "0xdeadbeef", test_0.shardId()), commonStats0 , null),
-                new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, "0xdeadbeef", test_1.shardId()), commonStats1 , null)
+                new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, test_0.shardId()), commonStats0 , null),
+                new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, test_1.shardId()), commonStats1 , null)
         };
         ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
         ImmutableOpenMap.Builder<ShardRouting, String> routingToPath = ImmutableOpenMap.builder();
@@ -143,11 +143,11 @@ public class DiskUsageTests extends ESTestCase {
         };
         NodeStats[] nodeStats = new NodeStats[] {
                 new NodeStats(new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, Version.CURRENT), 0,
-                        null,null,null,null,null,new FsInfo(0, node1FSInfo), null,null,null,null,null),
+                        null,null,null,null,null,new FsInfo(0, node1FSInfo), null,null,null,null,null, null),
                 new NodeStats(new DiscoveryNode("node_2", DummyTransportAddress.INSTANCE, Version.CURRENT), 0,
-                        null,null,null,null,null, new FsInfo(0, node2FSInfo), null,null,null,null,null),
+                        null,null,null,null,null, new FsInfo(0, node2FSInfo), null,null,null,null,null, null),
                 new NodeStats(new DiscoveryNode("node_3", DummyTransportAddress.INSTANCE, Version.CURRENT), 0,
-                        null,null,null,null,null, new FsInfo(0, node3FSInfo), null,null,null,null,null)
+                        null,null,null,null,null, new FsInfo(0, node3FSInfo), null,null,null,null,null, null)
         };
         InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvaiableUsages, newMostAvaiableUsages);
         DiskUsage leastNode_1 = newLeastAvaiableUsages.get("node_1");
@@ -184,11 +184,11 @@ public class DiskUsageTests extends ESTestCase {
         };
         NodeStats[] nodeStats = new NodeStats[] {
                 new NodeStats(new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, Version.CURRENT), 0,
-                        null,null,null,null,null,new FsInfo(0, node1FSInfo), null,null,null,null,null),
+                        null,null,null,null,null,new FsInfo(0, node1FSInfo), null,null,null,null,null, null),
                 new NodeStats(new DiscoveryNode("node_2", DummyTransportAddress.INSTANCE, Version.CURRENT), 0,
-                        null,null,null,null,null, new FsInfo(0, node2FSInfo), null,null,null,null,null),
+                        null,null,null,null,null, new FsInfo(0, node2FSInfo), null,null,null,null,null, null),
                 new NodeStats(new DiscoveryNode("node_3", DummyTransportAddress.INSTANCE, Version.CURRENT), 0,
-                        null,null,null,null,null, new FsInfo(0, node3FSInfo), null,null,null,null,null)
+                        null,null,null,null,null, new FsInfo(0, node3FSInfo), null,null,null,null,null, null)
         };
         InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvailableUsages, newMostAvailableUsages);
         DiskUsage leastNode_1 = newLeastAvailableUsages.get("node_1");
diff --git a/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java
index 78128fe30f2..9c004a95f3b 100644
--- a/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java
@@ -22,6 +22,7 @@ package org.elasticsearch.cluster;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
diff --git a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java
index 370f1464fd2..13b1d40b5d3 100644
--- a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java
@@ -235,7 +235,7 @@ public class NoMasterNodeIT extends ESIntegTestCase {
         ensureSearchable("test1", "test2");
 
         ClusterStateResponse clusterState = client().admin().cluster().prepareState().get();
-        logger.info("Cluster state:\n" + clusterState.getState().prettyPrint());
+        logger.info("Cluster state:\n{}", clusterState.getState().prettyPrint());
 
         internalCluster().stopRandomDataNode();
         assertTrue(awaitBusy(() -> {
diff --git a/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
new file mode 100644
index 00000000000..84c9e9f07a0
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.component.Lifecycle;
+import org.elasticsearch.common.component.LifecycleListener;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.transport.TransportServiceAdapter;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class NodeConnectionsServiceTests extends ESTestCase {
+
+    private static ThreadPool THREAD_POOL;
+    private MockTransport transport;
+    private TransportService transportService;
+
+    private List<DiscoveryNode> generateNodes() {
+        List<DiscoveryNode> nodes = new ArrayList<>();
+        for (int i = randomIntBetween(20, 50); i > 0; i--) {
+            final HashMap<String, String> attributes = new HashMap<>();
+            if (rarely()) {
+                attributes.put("client", "true");
+            } else {
+                attributes.put("master", "" + randomBoolean());
+                attributes.put("data", "" + randomBoolean());
+                attributes.put("ingest", "" + randomBoolean());
+            }
+            nodes.add(new DiscoveryNode("node_" + i, "" + i, DummyTransportAddress.INSTANCE, attributes, Version.CURRENT));
+        }
+        return nodes;
+    }
+
+    private ClusterState clusterStateFromNodes(List<DiscoveryNode> nodes) {
+        final DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
+        for (DiscoveryNode node : nodes) {
+            builder.put(node);
+        }
+        return ClusterState.builder(new ClusterName("test")).nodes(builder).build();
+    }
+
+    public void testConnectAndDisconnect() {
+        List<DiscoveryNode> nodes = generateNodes();
+        NodeConnectionsService service = new NodeConnectionsService(Settings.EMPTY, THREAD_POOL, transportService);
+
+        ClusterState current = clusterStateFromNodes(Collections.emptyList());
+        ClusterChangedEvent event = new ClusterChangedEvent("test", clusterStateFromNodes(randomSubsetOf(nodes)), current);
+
+        service.connectToAddedNodes(event);
+        assertConnected(event.nodesDelta().addedNodes());
+
+        service.disconnectFromRemovedNodes(event);
+        assertConnectedExactlyToNodes(event.state());
+
+        current = event.state();
+        event = new ClusterChangedEvent("test", clusterStateFromNodes(randomSubsetOf(nodes)), current);
+
+        service.connectToAddedNodes(event);
+        assertConnected(event.nodesDelta().addedNodes());
+
+        service.disconnectFromRemovedNodes(event);
+        assertConnectedExactlyToNodes(event.state());
+    }
+
+
+    public void testReconnect() {
+        List<DiscoveryNode> nodes = generateNodes();
+        NodeConnectionsService service = new NodeConnectionsService(Settings.EMPTY, THREAD_POOL, transportService);
+
+        ClusterState current = clusterStateFromNodes(Collections.emptyList());
+        ClusterChangedEvent event = new ClusterChangedEvent("test", clusterStateFromNodes(randomSubsetOf(nodes)), current);
+
+        transport.randomConnectionExceptions = true;
+
+        service.connectToAddedNodes(event);
+
+        for (int i = 0; i < 3; i++) {
+            // simulate disconnects
+            for (DiscoveryNode node : randomSubsetOf(nodes)) {
+                transport.disconnectFromNode(node);
+            }
+            service.new ConnectionChecker().run();
+        }
+
+        // disable exceptions so things can be restored
+        transport.randomConnectionExceptions = false;
+        service.new ConnectionChecker().run();
+        assertConnectedExactlyToNodes(event.state());
+    }
+
+    private void assertConnectedExactlyToNodes(ClusterState state) {
+        assertConnected(state.nodes());
+        assertThat(transport.connectedNodes.size(), equalTo(state.nodes().size()));
+    }
+
+    private void assertConnected(Iterable<DiscoveryNode> nodes) {
+        for (DiscoveryNode node : nodes) {
+            assertTrue("not connected to " + node, transport.connectedNodes.contains(node));
+        }
+    }
+
+    private void assertNotConnected(Iterable<DiscoveryNode> nodes) {
+        for (DiscoveryNode node : nodes) {
+            assertFalse("still connected to " + node, transport.connectedNodes.contains(node));
+        }
+    }
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        this.transport = new MockTransport();
+        transportService = new TransportService(transport, THREAD_POOL);
+        transportService.start();
+        transportService.acceptIncomingRequests();
+    }
+
+    @Override
+    @After
+    public void tearDown() throws Exception {
+        transportService.stop();
+        super.tearDown();
+    }
+
+    @AfterClass
+    public static void stopThreadPool() {
+        ThreadPool.terminate(THREAD_POOL, 30, TimeUnit.SECONDS);
+        THREAD_POOL = null;
+    }
+
+
+    final class MockTransport implements Transport {
+
+        Set<DiscoveryNode> connectedNodes = ConcurrentCollections.newConcurrentSet();
+        volatile boolean randomConnectionExceptions = false;
+
+        @Override
+        public void transportServiceAdapter(TransportServiceAdapter service) {
+
+        }
+
+        @Override
+        public BoundTransportAddress boundAddress() {
+            return null;
+        }
+
+        @Override
+        public Map<String, BoundTransportAddress> profileBoundAddresses() {
+            return null;
+        }
+
+        @Override
+        public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws Exception {
+            return new TransportAddress[0];
+        }
+
+        @Override
+        public boolean addressSupported(Class<? extends TransportAddress> address) {
+            return false;
+        }
+
+        @Override
+        public boolean nodeConnected(DiscoveryNode node) {
+            return connectedNodes.contains(node);
+        }
+
+        @Override
+        public void connectToNode(DiscoveryNode node) throws ConnectTransportException {
+            if (connectedNodes.contains(node) == false && randomConnectionExceptions && randomBoolean()) {
+                throw new ConnectTransportException(node, "simulated");
+            }
+            connectedNodes.add(node);
+        }
+
+        @Override
+        public void connectToNodeLight(DiscoveryNode node) throws ConnectTransportException {
+
+        }
+
+        @Override
+        public void disconnectFromNode(DiscoveryNode node) {
+            connectedNodes.remove(node);
+        }
+
+        @Override
+        public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request,
+                                TransportRequestOptions options) throws IOException, TransportException {
+
+        }
+
+        @Override
+        public long serverOpen() {
+            return 0;
+        }
+
+        @Override
+        public List<String> getLocalAddresses() {
+            return null;
+        }
+
+        @Override
+        public Lifecycle.State lifecycleState() {
+            return null;
+        }
+
+        @Override
+        public void addLifecycleListener(LifecycleListener listener) {
+
+        }
+
+        @Override
+        public void removeLifecycleListener(LifecycleListener listener) {
+
+        }
+
+        @Override
+        public Transport start() {
+            return null;
+        }
+
+        @Override
+        public Transport stop() {
+            return null;
+        }
+
+        @Override
+        public void close() {
+
+        }
+    }
+}
\ No newline at end of file
diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
index 6339c700eec..29ce8e7a636 100644
--- a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
@@ -27,6 +27,7 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
@@ -40,7 +41,6 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
-import org.elasticsearch.cluster.service.InternalClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESAllocationTestCase;
@@ -305,7 +305,7 @@ public class ShardFailedClusterStateTaskExecutorTests extends ESAllocationTestCa
             return randomSubsetOf(1, shards.toArray(new ShardRouting[0])).get(0);
         } else {
             return
-                    TestShardRouting.newShardRouting(shardRouting.index(), shardRouting.id(), InternalClusterService.generateNodeId(Settings.EMPTY), randomBoolean(), randomFrom(ShardRoutingState.values()));
+                    TestShardRouting.newShardRouting(shardRouting.index(), shardRouting.id(), DiscoveryNodeService.generateNodeId(Settings.EMPTY), randomBoolean(), randomFrom(ShardRoutingState.values()));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
index c4031edc2d6..be8984830e4 100644
--- a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
@@ -21,7 +21,6 @@ package org.elasticsearch.cluster.action.shard;
 
 import org.apache.lucene.index.CorruptIndexException;
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.NotMasterException;
@@ -34,10 +33,10 @@ import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.cluster.TestClusterService;
 import org.elasticsearch.test.transport.CapturingTransport;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.NodeDisconnectedException;
@@ -57,6 +56,8 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.LongConsumer;
 
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.Matchers.is;
@@ -67,7 +68,7 @@ public class ShardStateActionTests extends ESTestCase {
     private TestShardStateAction shardStateAction;
     private CapturingTransport transport;
     private TransportService transportService;
-    private TestClusterService clusterService;
+    private ClusterService clusterService;
 
     private static class TestShardStateAction extends ShardStateAction {
         public TestShardStateAction(Settings settings, ClusterService clusterService, TransportService transportService, AllocationService allocationService, RoutingService routingService) {
@@ -104,19 +105,22 @@ public class ShardStateActionTests extends ESTestCase {
     public void setUp() throws Exception {
         super.setUp();
         this.transport = new CapturingTransport();
-        clusterService = new TestClusterService(THREAD_POOL);
+        clusterService = createClusterService(THREAD_POOL);
         transportService = new TransportService(transport, THREAD_POOL);
         transportService.start();
         transportService.acceptIncomingRequests();
         shardStateAction = new TestShardStateAction(Settings.EMPTY, clusterService, transportService, null, null);
-        shardStateAction.setOnBeforeWaitForNewMasterAndRetry(() -> {});
-        shardStateAction.setOnAfterWaitForNewMasterAndRetry(() -> {});
+        shardStateAction.setOnBeforeWaitForNewMasterAndRetry(() -> {
+        });
+        shardStateAction.setOnAfterWaitForNewMasterAndRetry(() -> {
+        });
     }
 
     @Override
     @After
     public void tearDown() throws Exception {
-        transportService.stop();
+        clusterService.close();
+        transportService.close();
         super.tearDown();
     }
 
@@ -129,7 +133,7 @@ public class ShardStateActionTests extends ESTestCase {
     public void testSuccess() throws InterruptedException {
         final String index = "test";
 
-        clusterService.setState(ClusterStateCreationUtils.stateWithActivePrimary(index, true, randomInt(5)));
+        setState(clusterService, ClusterStateCreationUtils.stateWithActivePrimary(index, true, randomInt(5)));
 
         AtomicBoolean success = new AtomicBoolean();
         CountDownLatch latch = new CountDownLatch(1);
@@ -154,7 +158,7 @@ public class ShardStateActionTests extends ESTestCase {
         assertEquals(1, capturedRequests.length);
         // the request is a shard failed request
         assertThat(capturedRequests[0].request, is(instanceOf(ShardStateAction.ShardRoutingEntry.class)));
-        ShardStateAction.ShardRoutingEntry shardRoutingEntry = (ShardStateAction.ShardRoutingEntry)capturedRequests[0].request;
+        ShardStateAction.ShardRoutingEntry shardRoutingEntry = (ShardStateAction.ShardRoutingEntry) capturedRequests[0].request;
         // for the right shard
         assertEquals(shardRouting, shardRoutingEntry.getShardRouting());
         // sent to the master
@@ -169,17 +173,18 @@ public class ShardStateActionTests extends ESTestCase {
     public void testNoMaster() throws InterruptedException {
         final String index = "test";
 
-        clusterService.setState(ClusterStateCreationUtils.stateWithActivePrimary(index, true, randomInt(5)));
+        setState(clusterService, ClusterStateCreationUtils.stateWithActivePrimary(index, true, randomInt(5)));
 
         DiscoveryNodes.Builder noMasterBuilder = DiscoveryNodes.builder(clusterService.state().nodes());
         noMasterBuilder.masterNodeId(null);
-        clusterService.setState(ClusterState.builder(clusterService.state()).nodes(noMasterBuilder));
+        setState(clusterService, ClusterState.builder(clusterService.state()).nodes(noMasterBuilder));
 
         CountDownLatch latch = new CountDownLatch(1);
         AtomicInteger retries = new AtomicInteger();
         AtomicBoolean success = new AtomicBoolean();
 
-        setUpMasterRetryVerification(1, retries, latch, requestId -> {});
+        setUpMasterRetryVerification(1, retries, latch, requestId -> {
+        });
 
         ShardRouting failedShard = getRandomShardRouting(index);
         shardStateAction.shardFailed(failedShard, failedShard, "test", getSimulatedFailure(), new ShardStateAction.Listener() {
@@ -206,7 +211,7 @@ public class ShardStateActionTests extends ESTestCase {
     public void testMasterChannelException() throws InterruptedException {
         final String index = "test";
 
-        clusterService.setState(ClusterStateCreationUtils.stateWithActivePrimary(index, true, randomInt(5)));
+        setState(clusterService, ClusterStateCreationUtils.stateWithActivePrimary(index, true, randomInt(5)));
 
         CountDownLatch latch = new CountDownLatch(1);
         AtomicInteger retries = new AtomicInteger();
@@ -216,8 +221,8 @@ public class ShardStateActionTests extends ESTestCase {
         LongConsumer retryLoop = requestId -> {
             if (randomBoolean()) {
                 transport.handleRemoteError(
-                    requestId,
-                    randomFrom(new NotMasterException("simulated"), new Discovery.FailedToCommitClusterStateException("simulated")));
+                        requestId,
+                        randomFrom(new NotMasterException("simulated"), new Discovery.FailedToCommitClusterStateException("simulated")));
             } else {
                 if (randomBoolean()) {
                     transport.handleLocalError(requestId, new NodeNotConnectedException(null, "simulated"));
@@ -262,7 +267,7 @@ public class ShardStateActionTests extends ESTestCase {
     public void testUnhandledFailure() {
         final String index = "test";
 
-        clusterService.setState(ClusterStateCreationUtils.stateWithActivePrimary(index, true, randomInt(5)));
+        setState(clusterService, ClusterStateCreationUtils.stateWithActivePrimary(index, true, randomInt(5)));
 
         AtomicBoolean failure = new AtomicBoolean();
 
@@ -291,14 +296,14 @@ public class ShardStateActionTests extends ESTestCase {
     public void testShardNotFound() throws InterruptedException {
         final String index = "test";
 
-        clusterService.setState(ClusterStateCreationUtils.stateWithActivePrimary(index, true, randomInt(5)));
+        setState(clusterService, ClusterStateCreationUtils.stateWithActivePrimary(index, true, randomInt(5)));
 
         AtomicBoolean success = new AtomicBoolean();
         CountDownLatch latch = new CountDownLatch(1);
 
         ShardRouting failedShard = getRandomShardRouting(index);
         RoutingTable routingTable = RoutingTable.builder(clusterService.state().getRoutingTable()).remove(index).build();
-        clusterService.setState(ClusterState.builder(clusterService.state()).routingTable(routingTable));
+        setState(clusterService, ClusterState.builder(clusterService.state()).routingTable(routingTable));
         shardStateAction.shardFailed(failedShard, failedShard, "test", getSimulatedFailure(), new ShardStateAction.Listener() {
             @Override
             public void onSuccess() {
@@ -324,7 +329,7 @@ public class ShardStateActionTests extends ESTestCase {
     public void testNoLongerPrimaryShardException() throws InterruptedException {
         final String index = "test";
 
-        clusterService.setState(ClusterStateCreationUtils.stateWithActivePrimary(index, true, randomInt(5)));
+        setState(clusterService, ClusterStateCreationUtils.stateWithActivePrimary(index, true, randomInt(5)));
 
         ShardRouting failedShard = getRandomShardRouting(index);
 
@@ -349,7 +354,7 @@ public class ShardStateActionTests extends ESTestCase {
         });
 
         ShardStateAction.NoLongerPrimaryShardException catastrophicError =
-            new ShardStateAction.NoLongerPrimaryShardException(failedShard.shardId(), "source shard [" + sourceFailedShard + " is neither the local allocation nor the primary allocation");
+                new ShardStateAction.NoLongerPrimaryShardException(failedShard.shardId(), "source shard [" + sourceFailedShard + " is neither the local allocation nor the primary allocation");
         CapturingTransport.CapturedRequest[] capturedRequests = transport.getCapturedRequestsAndClear();
         transport.handleRemoteError(capturedRequests[0].requestId, catastrophicError);
 
@@ -371,7 +376,7 @@ public class ShardStateActionTests extends ESTestCase {
         shardStateAction.setOnBeforeWaitForNewMasterAndRetry(() -> {
             DiscoveryNodes.Builder masterBuilder = DiscoveryNodes.builder(clusterService.state().nodes());
             masterBuilder.masterNodeId(clusterService.state().nodes().masterNodes().iterator().next().value.id());
-            clusterService.setState(ClusterState.builder(clusterService.state()).nodes(masterBuilder));
+            setState(clusterService, ClusterState.builder(clusterService.state()).nodes(masterBuilder));
         });
 
         shardStateAction.setOnAfterWaitForNewMasterAndRetry(() -> verifyRetry(numberOfRetries, retries, latch, retryLoop));
diff --git a/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java b/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
index cc5ce05aca6..2016175a49c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
@@ -22,8 +22,10 @@ package org.elasticsearch.cluster.allocation;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -42,6 +44,7 @@ import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -226,9 +229,10 @@ public class ClusterRerouteIT extends ESIntegTestCase {
         assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).id()).get(0).state(), equalTo(ShardRoutingState.STARTED));
 
         client().prepareIndex("test", "type", "1").setSource("field", "value").setRefresh(true).execute().actionGet();
+        final Index index = resolveIndex("test");
 
         logger.info("--> closing all nodes");
-        Path[] shardLocation = internalCluster().getInstance(NodeEnvironment.class, node_1).availableShardPaths(new ShardId("test", "_na_", 0));
+        Path[] shardLocation = internalCluster().getInstance(NodeEnvironment.class, node_1).availableShardPaths(new ShardId(index, 0));
         assertThat(FileSystemUtils.exists(shardLocation), equalTo(true)); // make sure the data is there!
         internalCluster().closeNonSharedNodes(false); // don't wipe data directories the index needs to be there!
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
index 3562fa313ba..9c0f1014dcf 100644
--- a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
@@ -64,7 +64,7 @@ public class ClusterStateHealthTests extends ESTestCase {
             routingTable.add(indexRoutingTable);
         }
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable.build()).build();
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, IndicesOptions.strictExpand(), (String[]) null);
+        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, IndicesOptions.strictExpand(), (String[]) null);
         ClusterStateHealth clusterStateHealth = new ClusterStateHealth(clusterState, concreteIndices);
         logger.info("cluster status: {}, expected {}", clusterStateHealth.getStatus(), counter.status());
         clusterStateHealth = maybeSerialize(clusterStateHealth);
@@ -91,7 +91,7 @@ public class ClusterStateHealthTests extends ESTestCase {
         metaData.put(indexMetaData, true);
         routingTable.add(indexRoutingTable);
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable.build()).build();
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, IndicesOptions.strictExpand(), (String[]) null);
+        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, IndicesOptions.strictExpand(), (String[]) null);
         ClusterStateHealth clusterStateHealth = new ClusterStateHealth(clusterState, concreteIndices);
         clusterStateHealth = maybeSerialize(clusterStateHealth);
         // currently we have no cluster level validation failures as index validation issues are reported per index.
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
index 1c61292d87c..0e0c9fb442b 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
@@ -61,79 +61,79 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         IndicesOptions[] indicesOptions = new IndicesOptions[]{ IndicesOptions.strictExpandOpen(), IndicesOptions.strictExpand()};
         for (IndicesOptions options : indicesOptions) {
             IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options);
-            String[] results = indexNameExpressionResolver.concreteIndices(context, "foo");
+            String[] results = indexNameExpressionResolver.concreteIndexNames(context, "foo");
             assertEquals(1, results.length);
             assertEquals("foo", results[0]);
 
             try {
-                indexNameExpressionResolver.concreteIndices(context, "bar");
+                indexNameExpressionResolver.concreteIndexNames(context, "bar");
                 fail();
             } catch (IndexNotFoundException e) {
                 assertThat(e.getIndex().getName(), equalTo("bar"));
             }
 
-            results = indexNameExpressionResolver.concreteIndices(context, "foofoo", "foobar");
+            results = indexNameExpressionResolver.concreteIndexNames(context, "foofoo", "foobar");
             assertEquals(2, results.length);
             assertThat(results, arrayContainingInAnyOrder("foofoo", "foobar"));
 
-            results = indexNameExpressionResolver.concreteIndices(context, "foofoobar");
+            results = indexNameExpressionResolver.concreteIndexNames(context, "foofoobar");
             assertEquals(new HashSet<>(Arrays.asList("foo", "foobar")),
                          new HashSet<>(Arrays.asList(results)));
 
             try {
-                indexNameExpressionResolver.concreteIndices(context, "bar");
+                indexNameExpressionResolver.concreteIndexNames(context, "bar");
                 fail();
             } catch (IndexNotFoundException e) {
                 assertThat(e.getIndex().getName(), equalTo("bar"));
             }
 
             try {
-                indexNameExpressionResolver.concreteIndices(context, "foo", "bar");
+                indexNameExpressionResolver.concreteIndexNames(context, "foo", "bar");
                 fail();
             } catch (IndexNotFoundException e) {
                 assertThat(e.getIndex().getName(), equalTo("bar"));
             }
 
-            results = indexNameExpressionResolver.concreteIndices(context, "barbaz", "foobar");
+            results = indexNameExpressionResolver.concreteIndexNames(context, "barbaz", "foobar");
             assertEquals(2, results.length);
             assertThat(results, arrayContainingInAnyOrder("foofoo", "foobar"));
 
             try {
-                indexNameExpressionResolver.concreteIndices(context, "barbaz", "bar");
+                indexNameExpressionResolver.concreteIndexNames(context, "barbaz", "bar");
                 fail();
             } catch (IndexNotFoundException e) {
                 assertThat(e.getIndex().getName(), equalTo("bar"));
             }
 
-            results = indexNameExpressionResolver.concreteIndices(context, "baz*");
+            results = indexNameExpressionResolver.concreteIndexNames(context, "baz*");
             assertThat(results, emptyArray());
 
-            results = indexNameExpressionResolver.concreteIndices(context, "foo", "baz*");
+            results = indexNameExpressionResolver.concreteIndexNames(context, "foo", "baz*");
             assertEquals(1, results.length);
             assertEquals("foo", results[0]);
         }
 
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen());
-        String[] results = indexNameExpressionResolver.concreteIndices(context, Strings.EMPTY_ARRAY);
+        String[] results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(3, results.length);
 
-        results = indexNameExpressionResolver.concreteIndices(context, (String[])null);
+        results = indexNameExpressionResolver.concreteIndexNames(context, (String[])null);
         assertEquals(3, results.length);
 
         context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpand());
-        results = indexNameExpressionResolver.concreteIndices(context, Strings.EMPTY_ARRAY);
+        results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(4, results.length);
 
-        results = indexNameExpressionResolver.concreteIndices(context, (String[])null);
+        results = indexNameExpressionResolver.concreteIndexNames(context, (String[])null);
         assertEquals(4, results.length);
 
         context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen());
-        results = indexNameExpressionResolver.concreteIndices(context, "foofoo*");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foofoo*");
         assertEquals(3, results.length);
         assertThat(results, arrayContainingInAnyOrder("foo", "foobar", "foofoo"));
 
         context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpand());
-        results = indexNameExpressionResolver.concreteIndices(context, "foofoo*");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foofoo*");
         assertEquals(4, results.length);
         assertThat(results, arrayContainingInAnyOrder("foo", "foobar", "foofoo", "foofoo-closed"));
     }
@@ -150,57 +150,57 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         IndicesOptions[] indicesOptions = new IndicesOptions[]{ IndicesOptions.lenientExpandOpen(), lenientExpand};
         for (IndicesOptions options : indicesOptions) {
             IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options);
-            String[] results = indexNameExpressionResolver.concreteIndices(context, "foo");
+            String[] results = indexNameExpressionResolver.concreteIndexNames(context, "foo");
             assertEquals(1, results.length);
             assertEquals("foo", results[0]);
 
-            results = indexNameExpressionResolver.concreteIndices(context, "bar");
+            results = indexNameExpressionResolver.concreteIndexNames(context, "bar");
             assertThat(results, emptyArray());
 
-            results = indexNameExpressionResolver.concreteIndices(context, "foofoo", "foobar");
+            results = indexNameExpressionResolver.concreteIndexNames(context, "foofoo", "foobar");
             assertEquals(2, results.length);
             assertThat(results, arrayContainingInAnyOrder("foofoo", "foobar"));
 
-            results = indexNameExpressionResolver.concreteIndices(context, "foofoobar");
+            results = indexNameExpressionResolver.concreteIndexNames(context, "foofoobar");
             assertEquals(2, results.length);
             assertEquals(new HashSet<>(Arrays.asList("foo", "foobar")),
                          new HashSet<>(Arrays.asList(results)));
 
-            results = indexNameExpressionResolver.concreteIndices(context, "foo", "bar");
+            results = indexNameExpressionResolver.concreteIndexNames(context, "foo", "bar");
             assertEquals(1, results.length);
             assertThat(results, arrayContainingInAnyOrder("foo"));
 
-            results = indexNameExpressionResolver.concreteIndices(context, "barbaz", "foobar");
+            results = indexNameExpressionResolver.concreteIndexNames(context, "barbaz", "foobar");
             assertEquals(2, results.length);
             assertThat(results, arrayContainingInAnyOrder("foofoo", "foobar"));
 
-            results = indexNameExpressionResolver.concreteIndices(context, "barbaz", "bar");
+            results = indexNameExpressionResolver.concreteIndexNames(context, "barbaz", "bar");
             assertEquals(1, results.length);
             assertThat(results, arrayContainingInAnyOrder("foofoo"));
 
-            results = indexNameExpressionResolver.concreteIndices(context, "baz*");
+            results = indexNameExpressionResolver.concreteIndexNames(context, "baz*");
             assertThat(results, emptyArray());
 
-            results = indexNameExpressionResolver.concreteIndices(context, "foo", "baz*");
+            results = indexNameExpressionResolver.concreteIndexNames(context, "foo", "baz*");
             assertEquals(1, results.length);
             assertEquals("foo", results[0]);
         }
 
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
-        String[] results = indexNameExpressionResolver.concreteIndices(context, Strings.EMPTY_ARRAY);
+        String[] results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(3, results.length);
 
         context = new IndexNameExpressionResolver.Context(state, lenientExpand);
-        results = indexNameExpressionResolver.concreteIndices(context, Strings.EMPTY_ARRAY);
+        results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(Arrays.toString(results), 4, results.length);
 
         context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
-        results = indexNameExpressionResolver.concreteIndices(context,  "foofoo*");
+        results = indexNameExpressionResolver.concreteIndexNames(context,  "foofoo*");
         assertEquals(3, results.length);
         assertThat(results, arrayContainingInAnyOrder("foo", "foobar", "foofoo"));
 
         context = new IndexNameExpressionResolver.Context(state, lenientExpand);
-        results = indexNameExpressionResolver.concreteIndices(context, "foofoo*");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foofoo*");
         assertEquals(4, results.length);
         assertThat(results, arrayContainingInAnyOrder("foo", "foobar", "foofoo", "foofoo-closed"));
     }
@@ -219,26 +219,26 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         for (IndicesOptions options : indicesOptions) {
             IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options);
-            String[] results = indexNameExpressionResolver.concreteIndices(context, "foo");
+            String[] results = indexNameExpressionResolver.concreteIndexNames(context, "foo");
             assertEquals(1, results.length);
             assertEquals("foo", results[0]);
 
             try {
-                indexNameExpressionResolver.concreteIndices(context, "bar");
+                indexNameExpressionResolver.concreteIndexNames(context, "bar");
                 fail();
             } catch(IndexNotFoundException e) {
                 assertThat(e.getIndex().getName(), equalTo("bar"));
             }
 
             try {
-                indexNameExpressionResolver.concreteIndices(context, "baz*");
+                indexNameExpressionResolver.concreteIndexNames(context, "baz*");
                 fail();
             } catch (IndexNotFoundException e) {
                 assertThat(e.getIndex().getName(), equalTo("baz*"));
             }
 
             try {
-                indexNameExpressionResolver.concreteIndices(context, "foo", "baz*");
+                indexNameExpressionResolver.concreteIndexNames(context, "foo", "baz*");
                 fail();
             } catch (IndexNotFoundException e) {
                 assertThat(e.getIndex().getName(), equalTo("baz*"));
@@ -246,11 +246,11 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         }
 
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, expandOpen);
-        String[] results = indexNameExpressionResolver.concreteIndices(context, Strings.EMPTY_ARRAY);
+        String[] results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(3, results.length);
 
         context = new IndexNameExpressionResolver.Context(state, expand);
-        results = indexNameExpressionResolver.concreteIndices(context, Strings.EMPTY_ARRAY);
+        results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(4, results.length);
     }
 
@@ -264,60 +264,60 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         // Only closed
         IndicesOptions options = IndicesOptions.fromOptions(false, true, false, true);
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options);
-        String[] results = indexNameExpressionResolver.concreteIndices(context, Strings.EMPTY_ARRAY);
+        String[] results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(1, results.length);
         assertEquals("foo", results[0]);
 
-        results = indexNameExpressionResolver.concreteIndices(context, "foo*");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foo*");
         assertEquals(1, results.length);
         assertEquals("foo", results[0]);
 
         // no wildcards, so wildcard expansion don't apply
-        results = indexNameExpressionResolver.concreteIndices(context, "bar");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "bar");
         assertEquals(1, results.length);
         assertEquals("bar", results[0]);
 
         // Only open
         options = IndicesOptions.fromOptions(false, true, true, false);
         context = new IndexNameExpressionResolver.Context(state, options);
-        results = indexNameExpressionResolver.concreteIndices(context, Strings.EMPTY_ARRAY);
+        results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(2, results.length);
         assertThat(results, arrayContainingInAnyOrder("bar", "foobar"));
 
-        results = indexNameExpressionResolver.concreteIndices(context, "foo*");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foo*");
         assertEquals(1, results.length);
         assertEquals("foobar", results[0]);
 
-        results = indexNameExpressionResolver.concreteIndices(context, "bar");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "bar");
         assertEquals(1, results.length);
         assertEquals("bar", results[0]);
 
         // Open and closed
         options = IndicesOptions.fromOptions(false, true, true, true);
         context = new IndexNameExpressionResolver.Context(state, options);
-        results = indexNameExpressionResolver.concreteIndices(context, Strings.EMPTY_ARRAY);
+        results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(3, results.length);
         assertThat(results, arrayContainingInAnyOrder("bar", "foobar", "foo"));
 
-        results = indexNameExpressionResolver.concreteIndices(context, "foo*");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foo*");
         assertEquals(2, results.length);
         assertThat(results, arrayContainingInAnyOrder("foobar", "foo"));
 
-        results = indexNameExpressionResolver.concreteIndices(context, "bar");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "bar");
         assertEquals(1, results.length);
         assertEquals("bar", results[0]);
 
-        results = indexNameExpressionResolver.concreteIndices(context, "-foo*");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "-foo*");
         assertEquals(1, results.length);
         assertEquals("bar", results[0]);
 
-        results = indexNameExpressionResolver.concreteIndices(context, "-*");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "-*");
         assertEquals(0, results.length);
 
         options = IndicesOptions.fromOptions(false, false, true, true);
         context = new IndexNameExpressionResolver.Context(state, options);
         try {
-            indexNameExpressionResolver.concreteIndices(context, "-*");
+            indexNameExpressionResolver.concreteIndexNames(context, "-*");
             fail();
         } catch (IndexNotFoundException e) {
             assertThat(e.getResourceId().toString(), equalTo("[-*]"));
@@ -336,21 +336,21 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         {
             IndicesOptions noExpandLenient = IndicesOptions.fromOptions(true, true, false, false);
             IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, noExpandLenient);
-            String[] results = indexNameExpressionResolver.concreteIndices(context, "baz*");
+            String[] results = indexNameExpressionResolver.concreteIndexNames(context, "baz*");
             assertThat(results, emptyArray());
 
-            results = indexNameExpressionResolver.concreteIndices(context, "foo", "baz*");
+            results = indexNameExpressionResolver.concreteIndexNames(context, "foo", "baz*");
             assertEquals(1, results.length);
             assertEquals("foo", results[0]);
 
-            results = indexNameExpressionResolver.concreteIndices(context, "foofoobar");
+            results = indexNameExpressionResolver.concreteIndexNames(context, "foofoobar");
             assertEquals(2, results.length);
             assertThat(results, arrayContainingInAnyOrder("foo", "foobar"));
 
-            results = indexNameExpressionResolver.concreteIndices(context, (String[])null);
+            results = indexNameExpressionResolver.concreteIndexNames(context, (String[])null);
             assertEquals(0, results.length);
 
-            results = indexNameExpressionResolver.concreteIndices(context, Strings.EMPTY_ARRAY);
+            results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
             assertEquals(0, results.length);
         }
 
@@ -359,17 +359,17 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
             IndicesOptions noExpandDisallowEmpty = IndicesOptions.fromOptions(true, false, false, false);
             IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, noExpandDisallowEmpty);
             try {
-                indexNameExpressionResolver.concreteIndices(context, "baz*");
+                indexNameExpressionResolver.concreteIndexNames(context, "baz*");
                 fail();
             } catch (IndexNotFoundException e) {
                 assertThat(e.getIndex().getName(), equalTo("baz*"));
             }
 
-            String[] results = indexNameExpressionResolver.concreteIndices(context, "foo", "baz*");
+            String[] results = indexNameExpressionResolver.concreteIndexNames(context, "foo", "baz*");
             assertEquals(1, results.length);
             assertEquals("foo", results[0]);
 
-            results = indexNameExpressionResolver.concreteIndices(context, "foofoobar");
+            results = indexNameExpressionResolver.concreteIndexNames(context, "foofoobar");
             assertEquals(2, results.length);
             assertThat(results, arrayContainingInAnyOrder("foo", "foobar"));
         }
@@ -378,17 +378,17 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         {
             IndicesOptions noExpandErrorUnavailable = IndicesOptions.fromOptions(false, true, false, false);
             IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, noExpandErrorUnavailable);
-            String[] results = indexNameExpressionResolver.concreteIndices(context, "baz*");
+            String[] results = indexNameExpressionResolver.concreteIndexNames(context, "baz*");
             assertThat(results, emptyArray());
 
             try {
-                indexNameExpressionResolver.concreteIndices(context, "foo", "baz*");
+                indexNameExpressionResolver.concreteIndexNames(context, "foo", "baz*");
                 fail();
             } catch (IndexNotFoundException e) {
                 assertThat(e.getIndex().getName(), equalTo("baz*"));
             }
 
-            results = indexNameExpressionResolver.concreteIndices(context, "foofoobar");
+            results = indexNameExpressionResolver.concreteIndexNames(context, "foofoobar");
             assertEquals(2, results.length);
             assertThat(results, arrayContainingInAnyOrder("foo", "foobar"));
         }
@@ -398,20 +398,20 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
             IndicesOptions noExpandStrict = IndicesOptions.fromOptions(false, false, false, false);
             IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, noExpandStrict);
             try {
-                indexNameExpressionResolver.concreteIndices(context, "baz*");
+                indexNameExpressionResolver.concreteIndexNames(context, "baz*");
                 fail();
             } catch (IndexNotFoundException e) {
                 assertThat(e.getIndex().getName(), equalTo("baz*"));
             }
 
             try {
-                indexNameExpressionResolver.concreteIndices(context, "foo", "baz*");
+                indexNameExpressionResolver.concreteIndexNames(context, "foo", "baz*");
                 fail();
             } catch (IndexNotFoundException e) {
                 assertThat(e.getIndex().getName(), equalTo("baz*"));
             }
 
-            String[] results = indexNameExpressionResolver.concreteIndices(context, "foofoobar");
+            String[] results = indexNameExpressionResolver.concreteIndexNames(context, "foofoobar");
             assertEquals(2, results.length);
             assertThat(results, arrayContainingInAnyOrder("foo", "foobar"));
         }
@@ -429,7 +429,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         try {
             IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed());
-            indexNameExpressionResolver.concreteIndices(context, "baz*");
+            indexNameExpressionResolver.concreteIndexNames(context, "baz*");
             fail();
         } catch (IndexNotFoundException e) {
             assertThat(e.getIndex().getName(), equalTo("baz*"));
@@ -437,7 +437,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         try {
             IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed());
-            indexNameExpressionResolver.concreteIndices(context, "foo", "baz*");
+            indexNameExpressionResolver.concreteIndexNames(context, "foo", "baz*");
             fail();
         } catch (IndexNotFoundException e) {
             assertThat(e.getIndex().getName(), equalTo("baz*"));
@@ -445,7 +445,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         try {
             IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed());
-            indexNameExpressionResolver.concreteIndices(context, "foofoobar");
+            indexNameExpressionResolver.concreteIndexNames(context, "foofoobar");
             fail();
         } catch(IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("Alias [foofoobar] has more than one indices associated with it"));
@@ -453,7 +453,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         try {
             IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed());
-            indexNameExpressionResolver.concreteIndices(context, "foo", "foofoobar");
+            indexNameExpressionResolver.concreteIndexNames(context, "foo", "foofoobar");
             fail();
         } catch(IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("Alias [foofoobar] has more than one indices associated with it"));
@@ -461,7 +461,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         try {
             IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed());
-            indexNameExpressionResolver.concreteIndices(context, "foofoo-closed", "foofoobar");
+            indexNameExpressionResolver.concreteIndexNames(context, "foofoo-closed", "foofoobar");
             fail();
         } catch(IndexClosedException e) {
             assertThat(e.getMessage(), equalTo("closed"));
@@ -469,7 +469,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         }
 
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed());
-        String[] results = indexNameExpressionResolver.concreteIndices(context, "foo", "barbaz");
+        String[] results = indexNameExpressionResolver.concreteIndexNames(context, "foo", "barbaz");
         assertEquals(2, results.length);
         assertThat(results, arrayContainingInAnyOrder("foo", "foofoo"));
     }
@@ -479,18 +479,18 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         IndicesOptions options = IndicesOptions.strictExpandOpen();
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options);
-        String[] results = indexNameExpressionResolver.concreteIndices(context, Strings.EMPTY_ARRAY);
+        String[] results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertThat(results, emptyArray());
         try {
-            indexNameExpressionResolver.concreteIndices(context, "foo");
+            indexNameExpressionResolver.concreteIndexNames(context, "foo");
             fail();
         } catch (IndexNotFoundException e) {
             assertThat(e.getIndex().getName(), equalTo("foo"));
         }
-        results = indexNameExpressionResolver.concreteIndices(context, "foo*");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foo*");
         assertThat(results, emptyArray());
         try {
-            indexNameExpressionResolver.concreteIndices(context, "foo*", "bar");
+            indexNameExpressionResolver.concreteIndexNames(context, "foo*", "bar");
             fail();
         } catch (IndexNotFoundException e) {
             assertThat(e.getIndex().getName(), equalTo("bar"));
@@ -498,18 +498,18 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
 
         context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
-        results = indexNameExpressionResolver.concreteIndices(context, Strings.EMPTY_ARRAY);
+        results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertThat(results, emptyArray());
-        results = indexNameExpressionResolver.concreteIndices(context, "foo");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foo");
         assertThat(results, emptyArray());
-        results = indexNameExpressionResolver.concreteIndices(context, "foo*");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foo*");
         assertThat(results, emptyArray());
-        results = indexNameExpressionResolver.concreteIndices(context, "foo*", "bar");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foo*", "bar");
         assertThat(results, emptyArray());
 
         context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, false, true, false));
         try {
-            indexNameExpressionResolver.concreteIndices(context, Strings.EMPTY_ARRAY);
+            indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         } catch (IndexNotFoundException e) {
             assertThat(e.getResourceId().toString(), equalTo("[_all]"));
         }
@@ -527,7 +527,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen());
 
         try {
-            indexNameExpressionResolver.concreteIndices(context, "testZZZ");
+            indexNameExpressionResolver.concreteIndexNames(context, "testZZZ");
             fail("Expected IndexNotFoundException");
         } catch(IndexNotFoundException e) {
             assertThat(e.getMessage(), is("no such index"));
@@ -541,7 +541,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
 
-        assertThat(newHashSet(indexNameExpressionResolver.concreteIndices(context, "testXXX", "testZZZ")), equalTo(newHashSet("testXXX")));
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "testXXX", "testZZZ")), equalTo(newHashSet("testXXX")));
     }
 
     public void testConcreteIndicesIgnoreIndicesAllMissing() {
@@ -552,7 +552,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen());
 
         try {
-            indexNameExpressionResolver.concreteIndices(context, "testMo", "testMahdy");
+            indexNameExpressionResolver.concreteIndexNames(context, "testMo", "testMahdy");
             fail("Expected IndexNotFoundException");
         } catch(IndexNotFoundException e) {
             assertThat(e.getMessage(), is("no such index"));
@@ -565,7 +565,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
                 .put(indexBuilder("kuku"));
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
-        assertThat(newHashSet(indexNameExpressionResolver.concreteIndices(context, new String[]{})), equalTo(newHashSet("kuku", "testXXX")));
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, new String[]{})), equalTo(newHashSet("kuku", "testXXX")));
     }
 
     public void testConcreteIndicesWildcardExpansion() {
@@ -578,13 +578,13 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
 
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, false, false));
-        assertThat(newHashSet(indexNameExpressionResolver.concreteIndices(context, "testX*")), equalTo(new HashSet<String>()));
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "testX*")), equalTo(new HashSet<String>()));
         context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, true, false));
-        assertThat(newHashSet(indexNameExpressionResolver.concreteIndices(context, "testX*")), equalTo(newHashSet("testXXX", "testXXY")));
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "testX*")), equalTo(newHashSet("testXXX", "testXXY")));
         context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, false, true));
-        assertThat(newHashSet(indexNameExpressionResolver.concreteIndices(context, "testX*")), equalTo(newHashSet("testXYY")));
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "testX*")), equalTo(newHashSet("testXYY")));
         context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, true, true));
-        assertThat(newHashSet(indexNameExpressionResolver.concreteIndices(context, "testX*")), equalTo(newHashSet("testXXX", "testXXY", "testXYY")));
+        assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "testX*")), equalTo(newHashSet("testXXX", "testXXY", "testXYY")));
     }
 
     /**
@@ -610,7 +610,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
             // with no indices, asking for all indices should return empty list or exception, depending on indices options
             if (indicesOptions.allowNoIndices()) {
-                String[] concreteIndices = indexNameExpressionResolver.concreteIndices(context, allIndices);
+                String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(context, allIndices);
                 assertThat(concreteIndices, notNullValue());
                 assertThat(concreteIndices.length, equalTo(0));
             } else {
@@ -625,7 +625,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
             state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
             context = new IndexNameExpressionResolver.Context(state, indicesOptions);
             if (indicesOptions.expandWildcardsOpen() || indicesOptions.expandWildcardsClosed() || indicesOptions.allowNoIndices()) {
-                String[] concreteIndices = indexNameExpressionResolver.concreteIndices(context, allIndices);
+                String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(context, allIndices);
                 assertThat(concreteIndices, notNullValue());
                 int expectedNumberOfIndices = 0;
                 if (indicesOptions.expandWildcardsOpen()) {
@@ -646,7 +646,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
      */
     private void checkCorrectException(IndexNameExpressionResolver indexNameExpressionResolver, IndexNameExpressionResolver.Context context, String[] allIndices) {
         try {
-            indexNameExpressionResolver.concreteIndices(context, allIndices);
+            indexNameExpressionResolver.concreteIndexNames(context, allIndices);
             fail("wildcard expansion on should trigger IndexMissingException");
         } catch (IndexNotFoundException e) {
             // expected
@@ -668,12 +668,12 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
             // asking for non existing wildcard pattern should return empty list or exception
             if (indicesOptions.allowNoIndices()) {
-                String[] concreteIndices = indexNameExpressionResolver.concreteIndices(context, "Foo*");
+                String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(context, "Foo*");
                 assertThat(concreteIndices, notNullValue());
                 assertThat(concreteIndices.length, equalTo(0));
             } else {
                 try {
-                    indexNameExpressionResolver.concreteIndices(context, "Foo*");
+                    indexNameExpressionResolver.concreteIndexNames(context, "Foo*");
                     fail("expecting exception when result empty and allowNoIndicec=false");
                 } catch (IndexNotFoundException e) {
                     // expected exception
@@ -798,51 +798,51 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpenAndForbidClosed());
         try {
-            indexNameExpressionResolver.concreteIndices(context, "foo1-closed");
+            indexNameExpressionResolver.concreteIndexNames(context, "foo1-closed");
             fail("foo1-closed should be closed, but it is open");
         } catch (IndexClosedException e) {
             // expected
         }
 
         try {
-            indexNameExpressionResolver.concreteIndices(context, "foobar1-closed");
+            indexNameExpressionResolver.concreteIndexNames(context, "foobar1-closed");
             fail("foo1-closed should be closed, but it is open");
         } catch (IndexClosedException e) {
             // expected
         }
 
         context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, context.getOptions().allowNoIndices(), context.getOptions().expandWildcardsOpen(), context.getOptions().expandWildcardsClosed(), context.getOptions()));
-        String[] results = indexNameExpressionResolver.concreteIndices(context, "foo1-closed");
+        String[] results = indexNameExpressionResolver.concreteIndexNames(context, "foo1-closed");
         assertThat(results, emptyArray());
 
-        results = indexNameExpressionResolver.concreteIndices(context, "foobar1-closed");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foobar1-closed");
         assertThat(results, emptyArray());
 
         context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
-        results = indexNameExpressionResolver.concreteIndices(context, "foo1-closed");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foo1-closed");
         assertThat(results, arrayWithSize(1));
         assertThat(results, arrayContaining("foo1-closed"));
 
-        results = indexNameExpressionResolver.concreteIndices(context, "foobar1-closed");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foobar1-closed");
         assertThat(results, arrayWithSize(1));
         assertThat(results, arrayContaining("foo1-closed"));
 
         // testing an alias pointing to three indices:
         context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpenAndForbidClosed());
         try {
-            indexNameExpressionResolver.concreteIndices(context, "foobar2-closed");
+            indexNameExpressionResolver.concreteIndexNames(context, "foobar2-closed");
             fail("foo2-closed should be closed, but it is open");
         } catch (IndexClosedException e) {
             // expected
         }
 
         context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, context.getOptions().allowNoIndices(), context.getOptions().expandWildcardsOpen(), context.getOptions().expandWildcardsClosed(), context.getOptions()));
-        results = indexNameExpressionResolver.concreteIndices(context, "foobar2-closed");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foobar2-closed");
         assertThat(results, arrayWithSize(1));
         assertThat(results, arrayContaining("foo3"));
 
         context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
-        results = indexNameExpressionResolver.concreteIndices(context, "foobar2-closed");
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foobar2-closed");
         assertThat(results, arrayWithSize(3));
         assertThat(results, arrayContainingInAnyOrder("foo1-closed", "foo2-closed", "foo3"));
     }
@@ -855,7 +855,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
                 IndicesOptions.lenientExpandOpen(), IndicesOptions.strictExpandOpenAndForbidClosed()};
         for (IndicesOptions options : indicesOptions) {
             IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options);
-            String[] results = indexNameExpressionResolver.concreteIndices(context, "index1", "index1", "alias1");
+            String[] results = indexNameExpressionResolver.concreteIndexNames(context, "index1", "index1", "alias1");
             assertThat(results, equalTo(new String[]{"index1"}));
         }
     }
@@ -875,11 +875,11 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
 
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
-        String[] strings = indexNameExpressionResolver.concreteIndices(context, "alias-*");
+        String[] strings = indexNameExpressionResolver.concreteIndexNames(context, "alias-*");
         assertArrayEquals(new String[] {"test-0"}, strings);
 
         context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen());
-        strings = indexNameExpressionResolver.concreteIndices(context, "alias-*");
+        strings = indexNameExpressionResolver.concreteIndexNames(context, "alias-*");
 
         assertArrayEquals(new String[] {"test-0"}, strings);
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
index 58861585066..f7e8b18196f 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
@@ -41,11 +41,14 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase {
                 .put(IndexMetaData.builder("test1")
                         .settings(settings(Version.CURRENT))
                         .numberOfShards(1)
-                        .numberOfReplicas(2))
+                        .numberOfReplicas(2)
+                        .primaryTerm(0, 1))
                 .put(IndexMetaData.builder("test2")
                         .settings(settings(Version.CURRENT).put("setting1", "value1").put("setting2", "value2"))
                         .numberOfShards(2)
-                        .numberOfReplicas(3))
+                        .numberOfReplicas(3)
+                        .primaryTerm(0, 2)
+                        .primaryTerm(1, 2))
                 .put(IndexMetaData.builder("test3")
                         .settings(settings(Version.CURRENT))
                         .numberOfShards(1)
@@ -112,15 +115,15 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase {
                         .putAlias(newAliasMetaDataBuilder("alias1").filter(ALIAS_FILTER1))
                         .putAlias(newAliasMetaDataBuilder("alias2"))
                         .putAlias(newAliasMetaDataBuilder("alias4").filter(ALIAS_FILTER2)))
-                        .put(IndexTemplateMetaData.builder("foo")
-                                .template("bar")
-                                .order(1)
-                                .settings(settingsBuilder()
-                                        .put("setting1", "value1")
-                                        .put("setting2", "value2"))
-                                .putAlias(newAliasMetaDataBuilder("alias-bar1"))
-                                .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}"))
-                                .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar")))
+                .put(IndexTemplateMetaData.builder("foo")
+                        .template("bar")
+                        .order(1)
+                        .settings(settingsBuilder()
+                                .put("setting1", "value1")
+                                .put("setting2", "value2"))
+                        .putAlias(newAliasMetaDataBuilder("alias-bar1"))
+                        .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}"))
+                        .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar")))
                 .put(IndexMetaData.builder("test12")
                         .settings(settings(Version.CURRENT)
                                 .put("setting1", "value1")
@@ -133,15 +136,15 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase {
                         .putAlias(newAliasMetaDataBuilder("alias1").filter(ALIAS_FILTER1))
                         .putAlias(newAliasMetaDataBuilder("alias2"))
                         .putAlias(newAliasMetaDataBuilder("alias4").filter(ALIAS_FILTER2)))
-                        .put(IndexTemplateMetaData.builder("foo")
-                                .template("bar")
-                                .order(1)
-                                .settings(settingsBuilder()
-                                        .put("setting1", "value1")
-                                        .put("setting2", "value2"))
-                                .putAlias(newAliasMetaDataBuilder("alias-bar1"))
-                                .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}"))
-                                .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar")))
+                .put(IndexTemplateMetaData.builder("foo")
+                        .template("bar")
+                        .order(1)
+                        .settings(settingsBuilder()
+                                .put("setting1", "value1")
+                                .put("setting2", "value2"))
+                        .putAlias(newAliasMetaDataBuilder("alias-bar1"))
+                        .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}"))
+                        .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar")))
                 .build();
 
         String metaDataSource = MetaData.Builder.toXContent(metaData);
@@ -150,6 +153,7 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase {
         MetaData parsedMetaData = MetaData.Builder.fromXContent(XContentFactory.xContent(XContentType.JSON).createParser(metaDataSource));
 
         IndexMetaData indexMetaData = parsedMetaData.index("test1");
+        assertThat(indexMetaData.primaryTerm(0), equalTo(1L));
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
@@ -159,6 +163,8 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase {
         indexMetaData = parsedMetaData.index("test2");
         assertThat(indexMetaData.getNumberOfShards(), equalTo(2));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(3));
+        assertThat(indexMetaData.primaryTerm(0), equalTo(2L));
+        assertThat(indexMetaData.primaryTerm(1), equalTo(2L));
         assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
         assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5));
         assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java
index d9cf9f0d790..744477d6722 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java
@@ -47,6 +47,8 @@ public class WildcardExpressionResolverTests extends ESTestCase {
         assertThat(newHashSet(resolver.resolve(context, Arrays.asList("test*"))), equalTo(newHashSet("testXXX", "testXYY", "testYYY")));
         assertThat(newHashSet(resolver.resolve(context, Arrays.asList("testX*"))), equalTo(newHashSet("testXXX", "testXYY")));
         assertThat(newHashSet(resolver.resolve(context, Arrays.asList("testX*", "kuku"))), equalTo(newHashSet("testXXX", "testXYY", "kuku")));
+        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("*"))), equalTo(newHashSet("testXXX", "testXYY", "testYYY", "kuku")));
+        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("*", "-kuku"))), equalTo(newHashSet("testXXX", "testXYY", "testYYY")));
     }
 
     public void testConvertWildcardsTests() {
@@ -107,6 +109,18 @@ public class WildcardExpressionResolverTests extends ESTestCase {
         assertThat(newHashSet(resolver.resolve(context, Arrays.asList("*Y*X"))).size(), equalTo(0));
     }
 
+    public void testAll() {
+        MetaData.Builder mdBuilder = MetaData.builder()
+            .put(indexBuilder("testXXX"))
+            .put(indexBuilder("testXYY"))
+            .put(indexBuilder("testYYY"));
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
+        IndexNameExpressionResolver.WildcardExpressionResolver resolver = new IndexNameExpressionResolver.WildcardExpressionResolver();
+
+        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
+        assertThat(newHashSet(resolver.resolve(context, Arrays.asList("_all"))), equalTo(newHashSet("testXXX", "testXYY", "testYYY")));
+    }
+
     private IndexMetaData.Builder indexBuilder(String index) {
         return IndexMetaData.builder(index).settings(settings(Version.CURRENT).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java
index 94336d23623..da6f270a79d 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java
@@ -163,7 +163,7 @@ public class PrimaryAllocationIT extends ESIntegTestCase {
         for (IntObjectCursor<List<IndicesShardStoresResponse.StoreStatus>> shardStoreStatuses : storeStatuses) {
             int shardId = shardStoreStatuses.key;
             IndicesShardStoresResponse.StoreStatus storeStatus = randomFrom(shardStoreStatuses.value);
-            logger.info("--> adding allocation command for shard " + shardId);
+            logger.info("--> adding allocation command for shard {}", shardId);
             // force allocation based on node id
             if (useStaleReplica) {
                 rerouteBuilder.add(new AllocateStalePrimaryAllocationCommand("test", shardId, storeStatus.getNode().getId(), true));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java
new file mode 100644
index 00000000000..d9b74621cc5
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.health.ClusterStateHealth;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.node.DiscoveryNodes.Builder;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESAllocationTestCase;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
+import static org.elasticsearch.common.settings.Settings.settingsBuilder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class PrimaryTermsTests extends ESAllocationTestCase {
+
+    private static final String TEST_INDEX_1 = "test1";
+    private static final String TEST_INDEX_2 = "test2";
+    private RoutingTable testRoutingTable;
+    private int numberOfShards;
+    private int numberOfReplicas;
+    private final static Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
+    private AllocationService allocationService;
+    private ClusterState clusterState;
+
+    private final Map<String, long[]> primaryTermsPerIndex = new HashMap<>();
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        this.allocationService = createAllocationService(settingsBuilder()
+                .put("cluster.routing.allocation.node_concurrent_recoveries", Integer.MAX_VALUE) // don't limit recoveries
+                .put("cluster.routing.allocation.node_initial_primaries_recoveries", Integer.MAX_VALUE)
+                .build());
+        this.numberOfShards = randomIntBetween(1, 5);
+        this.numberOfReplicas = randomIntBetween(1, 5);
+        logger.info("Setup test with " + this.numberOfShards + " shards and " + this.numberOfReplicas + " replicas.");
+        this.primaryTermsPerIndex.clear();
+        MetaData metaData = MetaData.builder()
+                .put(createIndexMetaData(TEST_INDEX_1))
+                .put(createIndexMetaData(TEST_INDEX_2))
+                .build();
+
+        this.testRoutingTable = new RoutingTable.Builder()
+                .add(new IndexRoutingTable.Builder(metaData.index(TEST_INDEX_1).getIndex()).initializeAsNew(metaData.index(TEST_INDEX_1))
+                        .build())
+                .add(new IndexRoutingTable.Builder(metaData.index(TEST_INDEX_2).getIndex()).initializeAsNew(metaData.index(TEST_INDEX_2))
+                        .build())
+                .build();
+
+        this.clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData)
+                .routingTable(testRoutingTable).build();
+    }
+
+    /**
+     * puts primary shard routings into initializing state
+     */
+    private void initPrimaries() {
+        logger.info("adding " + (this.numberOfReplicas + 1) + " nodes and performing rerouting");
+        Builder discoBuilder = DiscoveryNodes.builder();
+        for (int i = 0; i < this.numberOfReplicas + 1; i++) {
+            discoBuilder = discoBuilder.put(newNode("node" + i));
+        }
+        this.clusterState = ClusterState.builder(clusterState).nodes(discoBuilder).build();
+        RoutingAllocation.Result rerouteResult = allocationService.reroute(clusterState, "reroute");
+        this.testRoutingTable = rerouteResult.routingTable();
+        assertThat(rerouteResult.changed(), is(true));
+        applyRerouteResult(rerouteResult);
+        primaryTermsPerIndex.keySet().forEach(this::incrementPrimaryTerm);
+    }
+
+    private void incrementPrimaryTerm(String index) {
+        final long[] primaryTerms = primaryTermsPerIndex.get(index);
+        for (int i = 0; i < primaryTerms.length; i++) {
+            primaryTerms[i]++;
+        }
+    }
+
+    private void incrementPrimaryTerm(String index, int shard) {
+        primaryTermsPerIndex.get(index)[shard]++;
+    }
+
+    private boolean startInitializingShards(String index) {
+        this.clusterState = ClusterState.builder(clusterState).routingTable(this.testRoutingTable).build();
+        final List<ShardRouting> startedShards = this.clusterState.getRoutingNodes().shardsWithState(index, INITIALIZING);
+        logger.info("start primary shards for index [{}]: {} ", index, startedShards);
+        RoutingAllocation.Result rerouteResult = allocationService.applyStartedShards(this.clusterState, startedShards);
+        applyRerouteResult(rerouteResult);
+        return rerouteResult.changed();
+    }
+
+    private void applyRerouteResult(RoutingAllocation.Result rerouteResult) {
+        ClusterState previousClusterState = this.clusterState;
+        ClusterState newClusterState = ClusterState.builder(previousClusterState).routingResult(rerouteResult).build();
+        ClusterState.Builder builder = ClusterState.builder(newClusterState).incrementVersion();
+        if (previousClusterState.routingTable() != newClusterState.routingTable()) {
+            builder.routingTable(RoutingTable.builder(newClusterState.routingTable()).version(newClusterState.routingTable().version() + 1)
+                    .build());
+        }
+        if (previousClusterState.metaData() != newClusterState.metaData()) {
+            builder.metaData(MetaData.builder(newClusterState.metaData()).version(newClusterState.metaData().version() + 1));
+        }
+        this.clusterState = builder.build();
+        this.testRoutingTable = rerouteResult.routingTable();
+        final ClusterStateHealth clusterHealth = new ClusterStateHealth(clusterState);
+        logger.info("applied reroute. active shards: p [{}], t [{}], init shards: [{}], relocating: [{}]",
+                clusterHealth.getActivePrimaryShards(), clusterHealth.getActiveShards(),
+                clusterHealth.getInitializingShards(), clusterHealth.getRelocatingShards());
+    }
+
+    private void failSomePrimaries(String index) {
+        this.clusterState = ClusterState.builder(clusterState).routingTable(this.testRoutingTable).build();
+        final IndexRoutingTable indexShardRoutingTable = testRoutingTable.index(index);
+        Set<Integer> shardIdsToFail = new HashSet<>();
+        for (int i = 1 + randomInt(numberOfShards - 1); i > 0; i--) {
+            shardIdsToFail.add(randomInt(numberOfShards - 1));
+        }
+        logger.info("failing primary shards {} for index [{}]", shardIdsToFail, index);
+        List<FailedRerouteAllocation.FailedShard> failedShards = new ArrayList<>();
+        for (int shard : shardIdsToFail) {
+            failedShards.add(new FailedRerouteAllocation.FailedShard(indexShardRoutingTable.shard(shard).primaryShard(), "test", null));
+            incrementPrimaryTerm(index, shard); // the primary failure should increment the primary term;
+        }
+        RoutingAllocation.Result rerouteResult = allocationService.applyFailedShards(this.clusterState, failedShards);
+        applyRerouteResult(rerouteResult);
+    }
+
+    private void addNodes() {
+        DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(clusterState.nodes());
+        final int newNodes = randomInt(10);
+        logger.info("adding [{}] nodes", newNodes);
+        for (int i = 0; i < newNodes; i++) {
+            nodesBuilder.put(newNode("extra_" + i));
+        }
+        this.clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build();
+        RoutingAllocation.Result rerouteResult = allocationService.reroute(this.clusterState, "nodes added");
+        applyRerouteResult(rerouteResult);
+
+    }
+
+    private IndexMetaData.Builder createIndexMetaData(String indexName) {
+        primaryTermsPerIndex.put(indexName, new long[numberOfShards]);
+        final IndexMetaData.Builder builder = new IndexMetaData.Builder(indexName)
+                .settings(DEFAULT_SETTINGS)
+                .numberOfReplicas(this.numberOfReplicas)
+                .numberOfShards(this.numberOfShards);
+        for (int i = 0; i < numberOfShards; i++) {
+            builder.primaryTerm(i, randomInt(200));
+            primaryTermsPerIndex.get(indexName)[i] = builder.primaryTerm(i);
+        }
+        return builder;
+    }
+
+    private void assertAllPrimaryTerm() {
+        primaryTermsPerIndex.keySet().forEach(this::assertPrimaryTerm);
+    }
+
+    private void assertPrimaryTerm(String index) {
+        final long[] terms = primaryTermsPerIndex.get(index);
+        final IndexMetaData indexMetaData = clusterState.metaData().index(index);
+        for (IndexShardRoutingTable shardRoutingTable : this.testRoutingTable.index(index)) {
+            final int shard = shardRoutingTable.shardId().id();
+            assertThat("primary term mismatch between indexMetaData of [" + index + "] and shard [" + shard + "]'s routing",
+                    indexMetaData.primaryTerm(shard), equalTo(terms[shard]));
+        }
+    }
+
+    public void testPrimaryTermMetaDataSync() {
+        assertAllPrimaryTerm();
+
+        initPrimaries();
+        assertAllPrimaryTerm();
+
+        startInitializingShards(TEST_INDEX_1);
+        assertAllPrimaryTerm();
+
+        startInitializingShards(TEST_INDEX_2);
+        assertAllPrimaryTerm();
+
+        // now start all replicas too
+        startInitializingShards(TEST_INDEX_1);
+        startInitializingShards(TEST_INDEX_2);
+        assertAllPrimaryTerm();
+
+        // relocations shouldn't change much
+        addNodes();
+        assertAllPrimaryTerm();
+        boolean changed = true;
+        while (changed) {
+            changed = startInitializingShards(TEST_INDEX_1);
+            assertAllPrimaryTerm();
+            changed |= startInitializingShards(TEST_INDEX_2);
+            assertAllPrimaryTerm();
+        }
+
+        // primary promotion
+        failSomePrimaries(TEST_INDEX_1);
+        assertAllPrimaryTerm();
+
+        // stablize cluster
+        changed = true;
+        while (changed) {
+            changed = startInitializingShards(TEST_INDEX_1);
+            assertAllPrimaryTerm();
+            changed |= startInitializingShards(TEST_INDEX_2);
+            assertAllPrimaryTerm();
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java
index 5c922f07e46..d35b896f705 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java
@@ -27,10 +27,10 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESAllocationTestCase;
-import org.elasticsearch.test.cluster.TestClusterService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
@@ -42,6 +42,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 
 /**
@@ -140,7 +142,7 @@ public class RoutingServiceTests extends ESAllocationTestCase {
      */
     public void testDelayedUnassignedScheduleRerouteAfterDelayedReroute() throws Exception {
         final ThreadPool testThreadPool = new ThreadPool(getTestName());
-
+        ClusterService clusterService = createClusterService(testThreadPool);
         try {
             MockAllocationService allocation = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator());
             MetaData metaData = MetaData.builder()
@@ -152,8 +154,8 @@ public class RoutingServiceTests extends ESAllocationTestCase {
             ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData)
                     .routingTable(RoutingTable.builder().addAsNew(metaData.index("short_delay")).addAsNew(metaData.index("long_delay")).build())
                     .nodes(DiscoveryNodes.builder()
-                    .put(newNode("node0", singletonMap("data", Boolean.FALSE.toString()))).localNodeId("node0").masterNodeId("node0")
-                    .put(newNode("node1")).put(newNode("node2")).put(newNode("node3")).put(newNode("node4"))).build();
+                            .put(newNode("node0", singletonMap("data", Boolean.FALSE.toString()))).localNodeId("node0").masterNodeId("node0")
+                            .put(newNode("node1")).put(newNode("node2")).put(newNode("node3")).put(newNode("node4"))).build();
             // allocate shards
             clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build();
             // start primaries
@@ -209,8 +211,7 @@ public class RoutingServiceTests extends ESAllocationTestCase {
 
             // manually trigger a clusterChanged event on routingService
             ClusterState newState = clusterState;
-            // create fake cluster service
-            TestClusterService clusterService = new TestClusterService(newState, testThreadPool);
+            setState(clusterService, newState);
             // create routing service, also registers listener on cluster service
             RoutingService routingService = new RoutingService(Settings.EMPTY, testThreadPool, clusterService, allocation);
             routingService.start(); // just so performReroute does not prematurely return
@@ -221,11 +222,12 @@ public class RoutingServiceTests extends ESAllocationTestCase {
             clusterService.addLast(event -> latch.countDown());
             // instead of clusterService calling clusterChanged, we call it directly here
             routingService.clusterChanged(new ClusterChangedEvent("test", newState, prevState));
-             // cluster service should have updated state and called routingService with clusterChanged
+            // cluster service should have updated state and called routingService with clusterChanged
             latch.await();
             // verify the registration has been set to the delay of longDelayReplica/longDelayUnassignedReplica
             assertThat(routingService.getMinDelaySettingAtLastSchedulingNanos(), equalTo(TimeValue.timeValueSeconds(10).nanos()));
         } finally {
+            clusterService.stop();
             terminate(testThreadPool);
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java
index 741d62d74e6..40e24338f00 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java
@@ -63,7 +63,7 @@ public class RoutingTableTests extends ESAllocationTestCase {
         this.numberOfReplicas = randomIntBetween(1, 5);
         this.shardsPerIndex = this.numberOfShards * (this.numberOfReplicas + 1);
         this.totalNumberOfShards = this.shardsPerIndex * 2;
-        logger.info("Setup test with " + this.numberOfShards + " shards and " + this.numberOfReplicas + " replicas.");
+        logger.info("Setup test with {} shards and {} replicas.", this.numberOfShards, this.numberOfReplicas);
         this.emptyRoutingTable = new RoutingTable.Builder().build();
         MetaData metaData = MetaData.builder()
                 .put(createIndexMetaData(TEST_INDEX_1))
@@ -81,7 +81,7 @@ public class RoutingTableTests extends ESAllocationTestCase {
      * puts primary shard routings into initializing state
      */
     private void initPrimaries() {
-        logger.info("adding " + (this.numberOfReplicas + 1) + " nodes and performing rerouting");
+        logger.info("adding {} nodes and performing rerouting", this.numberOfReplicas + 1);
         Builder discoBuilder = DiscoveryNodes.builder();
         for (int i = 0; i < this.numberOfReplicas + 1; i++) {
             discoBuilder = discoBuilder.put(newNode("node" + i));
@@ -95,7 +95,7 @@ public class RoutingTableTests extends ESAllocationTestCase {
 
     private void startInitializingShards(String index) {
         this.clusterState = ClusterState.builder(clusterState).routingTable(this.testRoutingTable).build();
-        logger.info("start primary shards for index " + index);
+        logger.info("start primary shards for index {}", index);
         RoutingAllocation.Result rerouteResult = ALLOCATION_SERVICE.applyStartedShards(this.clusterState, this.clusterState.getRoutingNodes().shardsWithState(index, INITIALIZING));
         this.clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         this.testRoutingTable = rerouteResult.routingTable();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java
index 91ba1f4999c..1c5f77ce408 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java
@@ -301,7 +301,7 @@ public class AddIncrementallyTests extends ESAllocationTestCase {
 
         RoutingTable routingTable = routingTableBuilder.build();
 
-        logger.info("start " + numberOfNodes + " nodes");
+        logger.info("start {} nodes", numberOfNodes);
         DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
         for (int i = 0; i < numberOfNodes; i++) {
             nodes.put(newNode("node" + i));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
index 8810fc47395..18f24504619 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
@@ -221,18 +221,10 @@ public class AwarenessAllocationTests extends ESAllocationTestCase {
         routingTable = strategy.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
 
-        for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(INITIALIZING)) {
-            logger.info(shard.toString());
-        }
-        for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(STARTED)) {
-            logger.info(shard.toString());
-        }
-        for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(RELOCATING)) {
-            logger.info(shard.toString());
-        }
-        for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(UNASSIGNED)) {
-            logger.info(shard.toString());
-        }
+        logger.info("Initializing shards: {}", clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+        logger.info("Started shards: {}", clusterState.getRoutingNodes().shardsWithState(STARTED));
+        logger.info("Relocating shards: {}", clusterState.getRoutingNodes().shardsWithState(RELOCATING));
+        logger.info("Unassigned shards: {}", clusterState.getRoutingNodes().shardsWithState(UNASSIGNED));
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(5));
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
index 707129578c9..56a66b52d6f 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
@@ -35,7 +35,6 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
-import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocators;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
@@ -311,29 +310,9 @@ public class BalanceConfigurationTests extends ESAllocationTestCase {
     public void testNoRebalanceOnPrimaryOverload() {
         Settings.Builder settings = settingsBuilder();
         AllocationService strategy = new AllocationService(settings.build(), randomAllocationDeciders(settings.build(),
-                new ClusterSettings(Settings.Builder.EMPTY_SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), getRandom()), new ShardsAllocators(settings.build(),
+                new ClusterSettings(Settings.Builder.EMPTY_SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), getRandom()),
                 NoopGatewayAllocator.INSTANCE, new ShardsAllocator() {
 
-            @Override
-            public boolean rebalance(RoutingAllocation allocation) {
-                return false;
-            }
-
-            @Override
-            public boolean moveShards(RoutingAllocation allocation) {
-                return false;
-            }
-
-            @Override
-            public void applyStartedShards(StartedRerouteAllocation allocation) {
-
-
-            }
-
-            @Override
-            public void applyFailedShards(FailedRerouteAllocation allocation) {
-            }
-
             /*
              *  // this allocator tries to rebuild this scenario where a rebalance is
              *  // triggered solely by the primary overload on node [1] where a shard
@@ -354,9 +333,8 @@ public class BalanceConfigurationTests extends ESAllocationTestCase {
                 --------[test][2], node[3], [P], s[STARTED]
                 --------[test][3], node[3], [P], s[STARTED]
                 ---- unassigned
-             */
-            @Override
-            public boolean allocateUnassigned(RoutingAllocation allocation) {
+            */
+            public boolean allocate(RoutingAllocation allocation) {
                 RoutingNodes.UnassignedShards unassigned = allocation.routingNodes().unassigned();
                 boolean changed = !unassigned.isEmpty();
                 ShardRouting[] drain = unassigned.drain();
@@ -403,7 +381,7 @@ public class BalanceConfigurationTests extends ESAllocationTestCase {
                 }
                 return changed;
             }
-        }), EmptyClusterInfoService.INSTANCE);
+        }, EmptyClusterInfoService.INSTANCE);
         MetaData.Builder metaDataBuilder = MetaData.builder();
         RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
         IndexMetaData.Builder indexMeta = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(1);
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java
index 1ba0c063255..0bd84413125 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java
@@ -30,7 +30,6 @@ import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
-import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESAllocationTestCase;
 
 import java.io.BufferedReader;
@@ -147,12 +146,12 @@ public abstract class CatAllocationTestCase extends ESAllocationTestCase {
             if (initializing.isEmpty()) {
                 break;
             }
-            logger.debug(initializing.toString());
+            logger.debug("Initializing shards: {}", initializing);
             numRelocations += initializing.size();
             routingTable = strategy.applyStartedShards(clusterState, initializing).routingTable();
             clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
         }
-        logger.debug("--> num relocations to get balance: " + numRelocations);
+        logger.debug("--> num relocations to get balance: {}", numRelocations);
         return clusterState;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
index 4e5be0f26b7..3ec8df5cea6 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
@@ -36,7 +36,7 @@ import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
-import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocators;
+import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
@@ -333,7 +333,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
         AllocationDeciders allocationDeciders = new AllocationDeciders(Settings.EMPTY, new AllocationDecider[] {new NodeVersionAllocationDecider(Settings.EMPTY)});
         AllocationService strategy = new MockAllocationService(Settings.EMPTY,
             allocationDeciders,
-            new ShardsAllocators(Settings.EMPTY, NoopGatewayAllocator.INSTANCE), EmptyClusterInfoService.INSTANCE);
+            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
         RoutingAllocation.Result result = strategy.reroute(state, new AllocationCommands(), true);
         // the two indices must stay as is, the replicas cannot move to oldNode2 because versions don't match
         state = ClusterState.builder(state).routingResult(result).build();
@@ -363,7 +363,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
             new NodeVersionAllocationDecider(Settings.EMPTY)});
         AllocationService strategy = new MockAllocationService(Settings.EMPTY,
             allocationDeciders,
-            new ShardsAllocators(Settings.EMPTY, NoopGatewayAllocator.INSTANCE), EmptyClusterInfoService.INSTANCE);
+            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
         RoutingAllocation.Result result = strategy.reroute(state, new AllocationCommands(), true);
 
         // Make sure that primary shards are only allocated on the new node
@@ -409,14 +409,16 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
                 String fromId = r.currentNodeId();
                 assertThat(fromId, notNullValue());
                 assertThat(toId, notNullValue());
-                logger.trace("From: " + fromId + " with Version: " + routingNodes.node(fromId).node().version() + " to: " + toId + " with Version: " + routingNodes.node(toId).node().version());
+                logger.trace("From: {} with Version: {} to: {} with Version: {}", fromId, routingNodes.node(fromId).node().version(),
+                    toId, routingNodes.node(toId).node().version());
                 assertTrue(routingNodes.node(toId).node().version().onOrAfter(routingNodes.node(fromId).node().version()));
             } else {
                 ShardRouting primary = routingNodes.activePrimary(r);
                 assertThat(primary, notNullValue());
                 String fromId = primary.currentNodeId();
                 String toId = r.relocatingNodeId();
-                logger.trace("From: " + fromId + " with Version: " + routingNodes.node(fromId).node().version() + " to: " + toId + " with Version: " + routingNodes.node(toId).node().version());
+                logger.trace("From: {} with Version: {} to: {} with Version: {}", fromId, routingNodes.node(fromId).node().version(),
+                    toId, routingNodes.node(toId).node().version());
                 assertTrue(routingNodes.node(toId).node().version().onOrAfter(routingNodes.node(fromId).node().version()));
             }
         }
@@ -428,7 +430,8 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
                 assertThat(primary, notNullValue());
                 String fromId = primary.currentNodeId();
                 String toId = r.currentNodeId();
-                logger.trace("From: " + fromId + " with Version: " + routingNodes.node(fromId).node().version() + " to: " + toId + " with Version: " + routingNodes.node(toId).node().version());
+                logger.trace("From: {} with Version: {} to: {} with Version: {}", fromId, routingNodes.node(fromId).node().version(),
+                    toId, routingNodes.node(toId).node().version());
                 assertTrue(routingNodes.node(toId).node().version().onOrAfter(routingNodes.node(fromId).node().version()));
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java
index 7e59ab8a6b4..b18ee32ff5e 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java
@@ -58,29 +58,31 @@ public class PrimaryElectionRoutingTests extends ESAllocationTestCase {
         ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("Adding two nodes and performing rerouting");
-        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
-        RoutingTable prevRoutingTable = routingTable;
-        routingTable = strategy.reroute(clusterState, "reroute").routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build();
+        RoutingAllocation.Result result = strategy.reroute(clusterState, "reroute");
+        clusterState = ClusterState.builder(clusterState).routingResult(result).build();
+
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node2"))).build();
+        result = strategy.reroute(clusterState, "reroute");
+        clusterState = ClusterState.builder(clusterState).routingResult(result).build();
 
         logger.info("Start the primary shard (on node1)");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
-        prevRoutingTable = routingTable;
-        routingTable = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING)).routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        result = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING));
+        clusterState = ClusterState.builder(clusterState).routingResult(result).build();
 
         logger.info("Start the backup shard (on node2)");
         routingNodes = clusterState.getRoutingNodes();
-        prevRoutingTable = routingTable;
-        routingTable = strategy.applyStartedShards(clusterState, routingNodes.node("node2").shardsWithState(INITIALIZING)).routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        result = strategy.applyStartedShards(clusterState, routingNodes.node("node2").shardsWithState(INITIALIZING));
+        clusterState = ClusterState.builder(clusterState).routingResult(result).build();
 
         logger.info("Adding third node and reroute and kill first node");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node3")).remove("node1")).build();
-        prevRoutingTable = routingTable;
-        routingTable = strategy.reroute(clusterState, "reroute").routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        RoutingTable prevRoutingTable = clusterState.routingTable();
+        result = strategy.reroute(clusterState, "reroute");
+        clusterState = ClusterState.builder(clusterState).routingResult(result).build();
         routingNodes = clusterState.getRoutingNodes();
+        routingTable = clusterState.routingTable();
 
         assertThat(prevRoutingTable != routingTable, equalTo(true));
         assertThat(routingTable.index("test").shards().size(), equalTo(1));
@@ -89,6 +91,7 @@ public class PrimaryElectionRoutingTests extends ESAllocationTestCase {
         assertThat(routingNodes.node("node3").numberOfShardsWithState(INITIALIZING), equalTo(1));
         // verify where the primary is
         assertThat(routingTable.index("test").shard(0).primaryShard().currentNodeId(), equalTo("node2"));
+        assertThat(clusterState.metaData().index("test").primaryTerm(0), equalTo(2L));
         assertThat(routingTable.index("test").shard(0).replicaShards().get(0).currentNodeId(), equalTo("node3"));
     }
 
@@ -110,16 +113,18 @@ public class PrimaryElectionRoutingTests extends ESAllocationTestCase {
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
         RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
+        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
 
         logger.info("Start the primary shards");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
         rerouteResult = allocation.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
-        clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
+        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(routingNodes.shardsWithState(STARTED).size(), equalTo(2));
         assertThat(routingNodes.shardsWithState(INITIALIZING).size(), equalTo(2));
+        assertThat(clusterState.metaData().index("test").primaryTerm(0), equalTo(1L));
+        assertThat(clusterState.metaData().index("test").primaryTerm(1), equalTo(1L));
 
         // now, fail one node, while the replica is initializing, and it also holds a primary
         logger.info("--> fail node with primary");
@@ -129,12 +134,13 @@ public class PrimaryElectionRoutingTests extends ESAllocationTestCase {
                 .put(newNode(nodeIdRemaining))
         ).build();
         rerouteResult = allocation.reroute(clusterState, "reroute");
-        clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
+        clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build();
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(routingNodes.shardsWithState(STARTED).size(), equalTo(1));
         assertThat(routingNodes.shardsWithState(INITIALIZING).size(), equalTo(1));
         assertThat(routingNodes.node(nodeIdRemaining).shardsWithState(INITIALIZING).get(0).primary(), equalTo(true));
+        assertThat(clusterState.metaData().index("test").primaryTerm(0), equalTo(2L));
 
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
index abc561a0916..0bdab7a1158 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
@@ -30,7 +30,7 @@ import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
-import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocators;
+import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
@@ -59,7 +59,7 @@ public class RandomAllocationDeciderTests extends ESAllocationTestCase {
         RandomAllocationDecider randomAllocationDecider = new RandomAllocationDecider(getRandom());
         AllocationService strategy = new AllocationService(settingsBuilder().build(), new AllocationDeciders(Settings.EMPTY,
                 new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY), new ReplicaAfterPrimaryActiveAllocationDecider(Settings.EMPTY),
-                        randomAllocationDecider))), new ShardsAllocators(NoopGatewayAllocator.INSTANCE), EmptyClusterInfoService.INSTANCE);
+                        randomAllocationDecider))), NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
         int indices = scaledRandomIntBetween(1, 20);
         Builder metaBuilder = MetaData.builder();
         int maxNumReplicas = 1;
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java
new file mode 100644
index 00000000000..38c15750423
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ShardStateIT extends ESIntegTestCase {
+
+    public void testPrimaryFailureIncreasesTerm() throws Exception {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        prepareCreate("test").setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).get();
+        ensureGreen();
+        assertPrimaryTerms(1, 1);
+
+        logger.info("--> disabling allocation to capture shard failure");
+        disableAllocation("test");
+
+        ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final int shard = randomBoolean() ? 0 : 1;
+        final String nodeId = state.routingTable().index("test").shard(shard).primaryShard().currentNodeId();
+        final String node = state.nodes().get(nodeId).name();
+        logger.info("--> failing primary of [{}] on node [{}]", shard, node);
+        IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node);
+        indicesService.indexService(resolveIndex("test")).getShard(shard).failShard("simulated test failure", null);
+
+        logger.info("--> waiting for a yellow index");
+        assertBusy(() -> assertThat(client().admin().cluster().prepareHealth().get().getStatus(), equalTo(ClusterHealthStatus.YELLOW)));
+
+        final long term0 = shard == 0 ? 2 : 1;
+        final long term1 = shard == 1 ? 2 : 1;
+        assertPrimaryTerms(term0, term1);
+
+        logger.info("--> enabling allocation");
+        enableAllocation("test");
+        ensureGreen();
+        assertPrimaryTerms(term0, term1);
+    }
+
+    protected void assertPrimaryTerms(long term0, long term1) {
+        for (String node : internalCluster().getNodeNames()) {
+            logger.debug("--> asserting primary terms terms on [{}]", node);
+            ClusterState state = client(node).admin().cluster().prepareState().setLocal(true).get().getState();
+            IndexMetaData metaData = state.metaData().index("test");
+            assertThat(metaData.primaryTerm(0), equalTo(term0));
+            assertThat(metaData.primaryTerm(1), equalTo(term1));
+            IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node);
+            IndexService indexService = indicesService.indexService(metaData.getIndex());
+            if (indexService != null) {
+                for (IndexShard shard : indexService) {
+                    assertThat("term mismatch for shard " + shard.shardId(),
+                        shard.getPrimaryTerm(), equalTo(metaData.primaryTerm(shard.shardId().id())));
+                }
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java
index 0830747a9dd..e220c8eb0f6 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java
@@ -212,7 +212,7 @@ public class ShardsLimitAllocationTests extends ESAllocationTestCase {
             assertThat(shardRouting.getIndexName(), equalTo("test1"));
         }
 
-        logger.info("update " + ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey() + " for test, see that things move");
+        logger.info("update {} for test, see that things move", ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey());
         metaData = MetaData.builder(metaData)
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)
                         .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5)
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
index 0855263dd06..928756fec01 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
@@ -39,7 +39,7 @@ import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocators;
+import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
@@ -65,10 +65,6 @@ import static org.hamcrest.Matchers.nullValue;
 
 public class DiskThresholdDeciderTests extends ESAllocationTestCase {
 
-    private static ShardsAllocators makeShardsAllocators() {
-        return new ShardsAllocators(NoopGatewayAllocator.INSTANCE);
-    }
-
     public void testDiskThreshold() {
         Settings diskSettings = settingsBuilder()
                 .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true)
@@ -109,7 +105,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, makeShardsAllocators(), cis);
+                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
@@ -194,7 +190,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, makeShardsAllocators(), cis);
+                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         routingTable = strategy.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
@@ -225,7 +221,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, makeShardsAllocators(), cis);
+                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         routingTable = strategy.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
@@ -305,7 +301,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, makeShardsAllocators(), cis);
+                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2))
@@ -362,7 +358,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, makeShardsAllocators(), cis);
+                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         routingTable = strategy.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
@@ -429,7 +425,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, makeShardsAllocators(), cis);
+                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         routingTable = strategy.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
@@ -460,7 +456,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, makeShardsAllocators(), cis);
+                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         routingTable = strategy.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
@@ -569,7 +565,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, makeShardsAllocators(), cis);
+                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
@@ -637,7 +633,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, makeShardsAllocators(), cis);
+                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
@@ -740,7 +736,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, makeShardsAllocators(), cis);
+                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
@@ -902,7 +898,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, makeShardsAllocators(), cis);
+                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
         // Ensure that the reroute call doesn't alter the routing table, since the first primary is relocating away
         // and therefor we will have sufficient disk space on node1.
         RoutingAllocation.Result result = strategy.reroute(clusterState, "reroute");
@@ -1003,7 +999,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
 
                 .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, makeShardsAllocators(), cis);
+                .build(), deciders, NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), cis);
         RoutingAllocation.Result result = strategy.reroute(clusterState, "reroute");
 
         assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().state(), equalTo(STARTED));
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
new file mode 100644
index 00000000000..cce3a873b7b
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
@@ -0,0 +1,652 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.service;
+
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
+import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalNodeMasterListener;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.component.LifecycleComponent;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.inject.Singleton;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.discovery.zen.ZenDiscovery;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.elasticsearch.common.settings.Settings.settingsBuilder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+
+/**
+ *
+ */
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
+@ESIntegTestCase.SuppressLocalMode
+public class ClusterServiceIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return pluginList(TestPlugin.class);
+    }
+
+    public void testAckedUpdateTask() throws Exception {
+        Settings settings = settingsBuilder()
+                .put("discovery.type", "local")
+                .build();
+        internalCluster().startNode(settings);
+        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
+
+        final AtomicBoolean allNodesAcked = new AtomicBoolean(false);
+        final AtomicBoolean ackTimeout = new AtomicBoolean(false);
+        final AtomicBoolean onFailure = new AtomicBoolean(false);
+        final AtomicBoolean executed = new AtomicBoolean(false);
+        final CountDownLatch latch = new CountDownLatch(1);
+        final CountDownLatch processedLatch = new CountDownLatch(1);
+        clusterService.submitStateUpdateTask("test", new AckedClusterStateUpdateTask<Void>(null, null) {
+            @Override
+            protected Void newResponse(boolean acknowledged) {
+                return null;
+            }
+
+            @Override
+            public boolean mustAck(DiscoveryNode discoveryNode) {
+                return true;
+            }
+
+            @Override
+            public void onAllNodesAcked(@Nullable Throwable t) {
+                allNodesAcked.set(true);
+                latch.countDown();
+            }
+
+            @Override
+            public void onAckTimeout() {
+                ackTimeout.set(true);
+                latch.countDown();
+            }
+
+            @Override
+            public TimeValue ackTimeout() {
+                return TimeValue.timeValueSeconds(10);
+            }
+
+            @Override
+            public TimeValue timeout() {
+                return TimeValue.timeValueSeconds(10);
+            }
+
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                processedLatch.countDown();
+            }
+
+            @Override
+            public ClusterState execute(ClusterState currentState) throws Exception {
+                executed.set(true);
+                return ClusterState.builder(currentState).build();
+            }
+
+            @Override
+            public void onFailure(String source, Throwable t) {
+                logger.error("failed to execute callback in test {}", t, source);
+                onFailure.set(true);
+                latch.countDown();
+            }
+        });
+
+        ensureGreen();
+        assertThat(latch.await(1, TimeUnit.SECONDS), equalTo(true));
+
+        assertThat(allNodesAcked.get(), equalTo(true));
+        assertThat(ackTimeout.get(), equalTo(false));
+        assertThat(executed.get(), equalTo(true));
+        assertThat(onFailure.get(), equalTo(false));
+
+        assertThat(processedLatch.await(1, TimeUnit.SECONDS), equalTo(true));
+    }
+
+    public void testAckedUpdateTaskSameClusterState() throws Exception {
+        Settings settings = settingsBuilder()
+                .put("discovery.type", "local")
+                .build();
+        internalCluster().startNode(settings);
+        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
+
+        final AtomicBoolean allNodesAcked = new AtomicBoolean(false);
+        final AtomicBoolean ackTimeout = new AtomicBoolean(false);
+        final AtomicBoolean onFailure = new AtomicBoolean(false);
+        final AtomicBoolean executed = new AtomicBoolean(false);
+        final CountDownLatch latch = new CountDownLatch(1);
+        final CountDownLatch processedLatch = new CountDownLatch(1);
+        clusterService.submitStateUpdateTask("test", new AckedClusterStateUpdateTask<Void>(null, null) {
+            @Override
+            protected Void newResponse(boolean acknowledged) {
+                return null;
+            }
+
+            @Override
+            public void onAllNodesAcked(@Nullable Throwable t) {
+                allNodesAcked.set(true);
+                latch.countDown();
+            }
+
+            @Override
+            public void onAckTimeout() {
+                ackTimeout.set(true);
+                latch.countDown();
+            }
+
+            @Override
+            public TimeValue ackTimeout() {
+                return TimeValue.timeValueSeconds(10);
+            }
+
+            @Override
+            public TimeValue timeout() {
+                return TimeValue.timeValueSeconds(10);
+            }
+
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                processedLatch.countDown();
+            }
+
+            @Override
+            public ClusterState execute(ClusterState currentState) throws Exception {
+                executed.set(true);
+                return currentState;
+            }
+
+            @Override
+            public void onFailure(String source, Throwable t) {
+                logger.error("failed to execute callback in test {}", t, source);
+                onFailure.set(true);
+                latch.countDown();
+            }
+        });
+
+        ensureGreen();
+        assertThat(latch.await(1, TimeUnit.SECONDS), equalTo(true));
+
+        assertThat(allNodesAcked.get(), equalTo(true));
+        assertThat(ackTimeout.get(), equalTo(false));
+        assertThat(executed.get(), equalTo(true));
+        assertThat(onFailure.get(), equalTo(false));
+
+        assertThat(processedLatch.await(1, TimeUnit.SECONDS), equalTo(true));
+    }
+
+    public void testAckedUpdateTaskNoAckExpected() throws Exception {
+        Settings settings = settingsBuilder()
+                .put("discovery.type", "local")
+                .build();
+        internalCluster().startNode(settings);
+        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
+
+        final AtomicBoolean allNodesAcked = new AtomicBoolean(false);
+        final AtomicBoolean ackTimeout = new AtomicBoolean(false);
+        final AtomicBoolean onFailure = new AtomicBoolean(false);
+        final AtomicBoolean executed = new AtomicBoolean(false);
+        final CountDownLatch latch = new CountDownLatch(1);
+        clusterService.submitStateUpdateTask("test", new AckedClusterStateUpdateTask<Void>(null, null) {
+            @Override
+            protected Void newResponse(boolean acknowledged) {
+                return null;
+            }
+
+            @Override
+            public boolean mustAck(DiscoveryNode discoveryNode) {
+                return false;
+            }
+
+            @Override
+            public void onAllNodesAcked(@Nullable Throwable t) {
+                allNodesAcked.set(true);
+                latch.countDown();
+            }
+
+            @Override
+            public void onAckTimeout() {
+                ackTimeout.set(true);
+                latch.countDown();
+            }
+
+            @Override
+            public TimeValue ackTimeout() {
+                return TimeValue.timeValueSeconds(10);
+            }
+
+            @Override
+            public TimeValue timeout() {
+                return TimeValue.timeValueSeconds(10);
+            }
+
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+            }
+
+            @Override
+            public ClusterState execute(ClusterState currentState) throws Exception {
+                executed.set(true);
+                return ClusterState.builder(currentState).build();
+            }
+
+            @Override
+            public void onFailure(String source, Throwable t) {
+                logger.error("failed to execute callback in test {}", t, source);
+                onFailure.set(true);
+                latch.countDown();
+            }
+        });
+
+        ensureGreen();
+        assertThat(latch.await(1, TimeUnit.SECONDS), equalTo(true));
+
+        assertThat(allNodesAcked.get(), equalTo(true));
+        assertThat(ackTimeout.get(), equalTo(false));
+        assertThat(executed.get(), equalTo(true));
+        assertThat(onFailure.get(), equalTo(false));
+    }
+
+    public void testAckedUpdateTaskTimeoutZero() throws Exception {
+        Settings settings = settingsBuilder()
+                .put("discovery.type", "local")
+                .build();
+        internalCluster().startNode(settings);
+        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
+
+        final AtomicBoolean allNodesAcked = new AtomicBoolean(false);
+        final AtomicBoolean ackTimeout = new AtomicBoolean(false);
+        final AtomicBoolean onFailure = new AtomicBoolean(false);
+        final AtomicBoolean executed = new AtomicBoolean(false);
+        final CountDownLatch latch = new CountDownLatch(1);
+        final CountDownLatch processedLatch = new CountDownLatch(1);
+        clusterService.submitStateUpdateTask("test", new AckedClusterStateUpdateTask<Void>(null, null) {
+            @Override
+            protected Void newResponse(boolean acknowledged) {
+                return null;
+            }
+
+            @Override
+            public boolean mustAck(DiscoveryNode discoveryNode) {
+                return false;
+            }
+
+            @Override
+            public void onAllNodesAcked(@Nullable Throwable t) {
+                allNodesAcked.set(true);
+                latch.countDown();
+            }
+
+            @Override
+            public void onAckTimeout() {
+                ackTimeout.set(true);
+                latch.countDown();
+            }
+
+            @Override
+            public TimeValue ackTimeout() {
+                return TimeValue.timeValueSeconds(0);
+            }
+
+            @Override
+            public TimeValue timeout() {
+                return TimeValue.timeValueSeconds(10);
+            }
+
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                processedLatch.countDown();
+            }
+
+            @Override
+            public ClusterState execute(ClusterState currentState) throws Exception {
+                executed.set(true);
+                return ClusterState.builder(currentState).build();
+            }
+
+            @Override
+            public void onFailure(String source, Throwable t) {
+                logger.error("failed to execute callback in test {}", t, source);
+                onFailure.set(true);
+                latch.countDown();
+            }
+        });
+
+        ensureGreen();
+        assertThat(latch.await(1, TimeUnit.SECONDS), equalTo(true));
+
+        assertThat(allNodesAcked.get(), equalTo(false));
+        assertThat(ackTimeout.get(), equalTo(true));
+        assertThat(executed.get(), equalTo(true));
+        assertThat(onFailure.get(), equalTo(false));
+
+        assertThat(processedLatch.await(1, TimeUnit.SECONDS), equalTo(true));
+    }
+
+    @TestLogging("_root:debug,action.admin.cluster.tasks:trace")
+    public void testPendingUpdateTask() throws Exception {
+        Settings settings = settingsBuilder()
+                .put("discovery.type", "local")
+                .build();
+        String node_0 = internalCluster().startNode(settings);
+        internalCluster().startNodeClient(settings);
+
+        final ClusterService clusterService = internalCluster().getInstance(ClusterService.class, node_0);
+        final CountDownLatch block1 = new CountDownLatch(1);
+        final CountDownLatch invoked1 = new CountDownLatch(1);
+        clusterService.submitStateUpdateTask("1", new ClusterStateUpdateTask() {
+            @Override
+            public ClusterState execute(ClusterState currentState) {
+                invoked1.countDown();
+                try {
+                    block1.await();
+                } catch (InterruptedException e) {
+                    fail();
+                }
+                return currentState;
+            }
+
+            @Override
+            public void onFailure(String source, Throwable t) {
+                invoked1.countDown();
+                fail();
+            }
+        });
+        invoked1.await();
+        final CountDownLatch invoked2 = new CountDownLatch(9);
+        for (int i = 2; i <= 10; i++) {
+            clusterService.submitStateUpdateTask(Integer.toString(i), new ClusterStateUpdateTask() {
+                @Override
+                public ClusterState execute(ClusterState currentState) {
+                    return currentState;
+                }
+
+                @Override
+                public void onFailure(String source, Throwable t) {
+                    fail();
+                }
+
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    invoked2.countDown();
+                }
+            });
+        }
+
+        // there might be other tasks in this node, make sure to only take the ones we add into account in this test
+
+        // The tasks can be re-ordered, so we need to check out-of-order
+        Set<String> controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9", "10"));
+        List<PendingClusterTask> pendingClusterTasks = clusterService.pendingTasks();
+        assertThat(pendingClusterTasks.size(), greaterThanOrEqualTo(10));
+        assertThat(pendingClusterTasks.get(0).getSource().string(), equalTo("1"));
+        assertThat(pendingClusterTasks.get(0).isExecuting(), equalTo(true));
+        for (PendingClusterTask task : pendingClusterTasks) {
+            controlSources.remove(task.getSource().string());
+        }
+        assertTrue(controlSources.isEmpty());
+
+        controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9", "10"));
+        PendingClusterTasksResponse response = internalCluster().clientNodeClient().admin().cluster().preparePendingClusterTasks().get();
+        assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(10));
+        assertThat(response.pendingTasks().get(0).getSource().string(), equalTo("1"));
+        assertThat(response.pendingTasks().get(0).isExecuting(), equalTo(true));
+        for (PendingClusterTask task : response) {
+            controlSources.remove(task.getSource().string());
+        }
+        assertTrue(controlSources.isEmpty());
+        block1.countDown();
+        invoked2.await();
+
+        // whenever we test for no tasks, we need to awaitBusy since this is a live node
+        assertTrue(awaitBusy(() -> clusterService.pendingTasks().isEmpty()));
+        waitNoPendingTasksOnAll();
+
+        final CountDownLatch block2 = new CountDownLatch(1);
+        final CountDownLatch invoked3 = new CountDownLatch(1);
+        clusterService.submitStateUpdateTask("1", new ClusterStateUpdateTask() {
+            @Override
+            public ClusterState execute(ClusterState currentState) {
+                invoked3.countDown();
+                try {
+                    block2.await();
+                } catch (InterruptedException e) {
+                    fail();
+                }
+                return currentState;
+            }
+
+            @Override
+            public void onFailure(String source, Throwable t) {
+                invoked3.countDown();
+                fail();
+            }
+        });
+        invoked3.await();
+
+        for (int i = 2; i <= 5; i++) {
+            clusterService.submitStateUpdateTask(Integer.toString(i), new ClusterStateUpdateTask() {
+                @Override
+                public ClusterState execute(ClusterState currentState) {
+                    return currentState;
+                }
+
+                @Override
+                public void onFailure(String source, Throwable t) {
+                    fail();
+                }
+            });
+        }
+        Thread.sleep(100);
+
+        pendingClusterTasks = clusterService.pendingTasks();
+        assertThat(pendingClusterTasks.size(), greaterThanOrEqualTo(5));
+        controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5"));
+        for (PendingClusterTask task : pendingClusterTasks) {
+            controlSources.remove(task.getSource().string());
+        }
+        assertTrue(controlSources.isEmpty());
+
+        response = internalCluster().clientNodeClient().admin().cluster().preparePendingClusterTasks().get();
+        assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(5));
+        controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5"));
+        for (PendingClusterTask task : response) {
+            if (controlSources.remove(task.getSource().string())) {
+                assertThat(task.getTimeInQueueInMillis(), greaterThan(0L));
+            }
+        }
+        assertTrue(controlSources.isEmpty());
+        block2.countDown();
+    }
+
+    public void testLocalNodeMasterListenerCallbacks() throws Exception {
+        Settings settings = settingsBuilder()
+                .put("discovery.type", "zen")
+                .put("discovery.zen.minimum_master_nodes", 1)
+                .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "400ms")
+                .put("discovery.initial_state_timeout", "500ms")
+                .build();
+
+        String node_0 = internalCluster().startNode(settings);
+        ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
+        MasterAwareService testService = internalCluster().getInstance(MasterAwareService.class);
+
+        ClusterHealthResponse clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID)
+                .setWaitForNodes("1").get();
+        assertThat(clusterHealth.isTimedOut(), equalTo(false));
+
+        // the first node should be a master as the minimum required is 1
+        assertThat(clusterService.state().nodes().masterNode(), notNullValue());
+        assertThat(clusterService.state().nodes().localNodeMaster(), is(true));
+        assertThat(testService.master(), is(true));
+
+        String node_1 = internalCluster().startNode(settings);
+        final ClusterService clusterService1 = internalCluster().getInstance(ClusterService.class, node_1);
+        MasterAwareService testService1 = internalCluster().getInstance(MasterAwareService.class, node_1);
+
+        clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("2").get();
+        assertThat(clusterHealth.isTimedOut(), equalTo(false));
+
+        // the second node should not be the master as node1 is already the master.
+        assertThat(clusterService1.state().nodes().localNodeMaster(), is(false));
+        assertThat(testService1.master(), is(false));
+
+        internalCluster().stopCurrentMasterNode();
+        clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("1").get();
+        assertThat(clusterHealth.isTimedOut(), equalTo(false));
+
+        // now that node0 is closed, node1 should be elected as master
+        assertThat(clusterService1.state().nodes().localNodeMaster(), is(true));
+        assertThat(testService1.master(), is(true));
+
+        // start another node and set min_master_node
+        internalCluster().startNode(Settings.builder().put(settings));
+        assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());
+
+        Settings transientSettings = settingsBuilder()
+                .put("discovery.zen.minimum_master_nodes", 2)
+                .build();
+        client().admin().cluster().prepareUpdateSettings().setTransientSettings(transientSettings).get();
+
+        // and shutdown the second node
+        internalCluster().stopRandomNonMasterNode();
+
+        // there should not be any master as the minimum number of required eligible masters is not met
+        awaitBusy(() -> clusterService1.state().nodes().masterNode() == null &&
+                clusterService1.state().status() == ClusterState.ClusterStateStatus.APPLIED);
+        assertThat(testService1.master(), is(false));
+
+        // bring the node back up
+        String node_2 = internalCluster().startNode(Settings.builder().put(settings).put(transientSettings));
+        ClusterService clusterService2 = internalCluster().getInstance(ClusterService.class, node_2);
+        MasterAwareService testService2 = internalCluster().getInstance(MasterAwareService.class, node_2);
+
+        // make sure both nodes see each other otherwise the masternode below could be null if node 2 is master and node 1 did'r receive
+        // the updated cluster state...
+        assertThat(internalCluster().client(node_1).admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setLocal(true)
+                .setWaitForNodes("2").get().isTimedOut(), is(false));
+        assertThat(internalCluster().client(node_2).admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setLocal(true)
+                .setWaitForNodes("2").get().isTimedOut(), is(false));
+
+        // now that we started node1 again, a new master should be elected
+        assertThat(clusterService2.state().nodes().masterNode(), is(notNullValue()));
+        if (node_2.equals(clusterService2.state().nodes().masterNode().name())) {
+            assertThat(testService1.master(), is(false));
+            assertThat(testService2.master(), is(true));
+        } else {
+            assertThat(testService1.master(), is(true));
+            assertThat(testService2.master(), is(false));
+        }
+    }
+
+    public static class TestPlugin extends Plugin {
+
+        @Override
+        public String name() {
+            return "test plugin";
+        }
+
+        @Override
+        public String description() {
+            return "test plugin";
+        }
+
+        @Override
+        public Collection<Class<? extends LifecycleComponent>> nodeServices() {
+            List<Class<? extends LifecycleComponent>> services = new ArrayList<>(1);
+            services.add(MasterAwareService.class);
+            return services;
+        }
+    }
+
+    @Singleton
+    public static class MasterAwareService extends AbstractLifecycleComponent<MasterAwareService> implements LocalNodeMasterListener {
+
+        private final ClusterService clusterService;
+        private volatile boolean master;
+
+        @Inject
+        public MasterAwareService(Settings settings, ClusterService clusterService) {
+            super(settings);
+            clusterService.add(this);
+            this.clusterService = clusterService;
+            logger.info("initialized test service");
+        }
+
+        @Override
+        public void onMaster() {
+            logger.info("on master [{}]", clusterService.localNode());
+            master = true;
+        }
+
+        @Override
+        public void offMaster() {
+            logger.info("off master [{}]", clusterService.localNode());
+            master = false;
+        }
+
+        public boolean master() {
+            return master;
+        }
+
+        @Override
+        protected void doStart() {
+        }
+
+        @Override
+        protected void doStop() {
+        }
+
+        @Override
+        protected void doClose() {
+        }
+
+        @Override
+        public String executorName() {
+            return ThreadPool.Names.SAME;
+        }
+
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
new file mode 100644
index 00000000000..a62f99ab459
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
@@ -0,0 +1,804 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.service;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateTaskConfig;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor;
+import org.elasticsearch.cluster.ClusterStateTaskListener;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.NodeConnectionsService;
+import org.elasticsearch.cluster.block.ClusterBlocks;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.OperationRouting;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.MockLogAppender;
+import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.common.settings.Settings.settingsBuilder;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class ClusterServiceTests extends ESTestCase {
+
+    static ThreadPool threadPool;
+    TimedClusterService clusterService;
+
+    @BeforeClass
+    public static void createThreadPool() {
+        threadPool = new ThreadPool(ClusterServiceTests.class.getName());
+    }
+
+    @AfterClass
+    public static void stopThreadPool() {
+        if (threadPool != null) {
+            threadPool.shutdownNow();
+            threadPool = null;
+        }
+    }
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        clusterService = createTimedClusterService(true);
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        clusterService.close();
+        super.tearDown();
+    }
+
+    TimedClusterService createTimedClusterService(boolean makeMaster) throws InterruptedException {
+        TimedClusterService timedClusterService = new TimedClusterService(Settings.EMPTY, null,
+                new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                threadPool, new ClusterName("ClusterServiceTests"));
+        timedClusterService.setLocalNode(new DiscoveryNode("node1", DummyTransportAddress.INSTANCE, Version.CURRENT));
+        timedClusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) {
+            @Override
+            public void connectToAddedNodes(ClusterChangedEvent event) {
+                // skip
+            }
+
+            @Override
+            public void disconnectFromRemovedNodes(ClusterChangedEvent event) {
+                // skip
+            }
+        });
+        timedClusterService.setClusterStatePublisher((event, ackListener) -> {
+        });
+        timedClusterService.start();
+        ClusterState state = timedClusterService.state();
+        final DiscoveryNodes nodes = state.nodes();
+        final DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(nodes)
+                .masterNodeId(makeMaster ? nodes.localNodeId() : null);
+        state = ClusterState.builder(state).blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK)
+                .nodes(nodesBuilder).build();
+        setState(timedClusterService, state);
+        return timedClusterService;
+    }
+
+    public void testTimeoutUpdateTask() throws Exception {
+        final CountDownLatch block = new CountDownLatch(1);
+        clusterService.submitStateUpdateTask("test1", new ClusterStateUpdateTask() {
+            @Override
+            public ClusterState execute(ClusterState currentState) {
+                try {
+                    block.await();
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+                return currentState;
+            }
+
+            @Override
+            public void onFailure(String source, Throwable t) {
+                throw new RuntimeException(t);
+            }
+        });
+
+        final CountDownLatch timedOut = new CountDownLatch(1);
+        final AtomicBoolean executeCalled = new AtomicBoolean();
+        clusterService.submitStateUpdateTask("test2", new ClusterStateUpdateTask() {
+            @Override
+            public TimeValue timeout() {
+                return TimeValue.timeValueMillis(2);
+            }
+
+            @Override
+            public void onFailure(String source, Throwable t) {
+                timedOut.countDown();
+            }
+
+            @Override
+            public ClusterState execute(ClusterState currentState) {
+                executeCalled.set(true);
+                return currentState;
+            }
+
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+            }
+        });
+
+        timedOut.await();
+        block.countDown();
+        final CountDownLatch allProcessed = new CountDownLatch(1);
+        clusterService.submitStateUpdateTask("test3", new ClusterStateUpdateTask() {
+            @Override
+            public void onFailure(String source, Throwable t) {
+                throw new RuntimeException(t);
+            }
+
+            @Override
+            public ClusterState execute(ClusterState currentState) {
+                allProcessed.countDown();
+                return currentState;
+            }
+
+        });
+        allProcessed.await(); // executed another task to double check that execute on the timed out update task is not called...
+        assertThat(executeCalled.get(), equalTo(false));
+    }
+
+
+    public void testMasterAwareExecution() throws Exception {
+        ClusterService nonMaster = createTimedClusterService(false);
+
+        final boolean[] taskFailed = {false};
+        final CountDownLatch latch1 = new CountDownLatch(1);
+        nonMaster.submitStateUpdateTask("test", new ClusterStateUpdateTask() {
+            @Override
+            public ClusterState execute(ClusterState currentState) throws Exception {
+                latch1.countDown();
+                return currentState;
+            }
+
+            @Override
+            public void onFailure(String source, Throwable t) {
+                taskFailed[0] = true;
+                latch1.countDown();
+            }
+        });
+
+        latch1.await();
+        assertTrue("cluster state update task was executed on a non-master", taskFailed[0]);
+
+        taskFailed[0] = true;
+        final CountDownLatch latch2 = new CountDownLatch(1);
+        nonMaster.submitStateUpdateTask("test", new ClusterStateUpdateTask() {
+            @Override
+            public boolean runOnlyOnMaster() {
+                return false;
+            }
+
+            @Override
+            public ClusterState execute(ClusterState currentState) throws Exception {
+                taskFailed[0] = false;
+                latch2.countDown();
+                return currentState;
+            }
+
+            @Override
+            public void onFailure(String source, Throwable t) {
+                taskFailed[0] = true;
+                latch2.countDown();
+            }
+        });
+        latch2.await();
+        assertFalse("non-master cluster state update task was not executed", taskFailed[0]);
+
+        nonMaster.close();
+    }
+
+    /*
+   * test that a listener throwing an exception while handling a
+   * notification does not prevent publication notification to the
+   * executor
+   */
+    public void testClusterStateTaskListenerThrowingExceptionIsOkay() throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(1);
+        AtomicBoolean published = new AtomicBoolean();
+
+        clusterService.submitStateUpdateTask(
+                "testClusterStateTaskListenerThrowingExceptionIsOkay",
+                new Object(),
+                ClusterStateTaskConfig.build(Priority.NORMAL),
+                new ClusterStateTaskExecutor<Object>() {
+                    @Override
+                    public boolean runOnlyOnMaster() {
+                        return false;
+                    }
+
+                    @Override
+                    public BatchResult<Object> execute(ClusterState currentState, List<Object> tasks) throws Exception {
+                        ClusterState newClusterState = ClusterState.builder(currentState).build();
+                        return BatchResult.builder().successes(tasks).build(newClusterState);
+                    }
+
+                    @Override
+                    public void clusterStatePublished(ClusterState newClusterState) {
+                        published.set(true);
+                        latch.countDown();
+                    }
+                },
+                new ClusterStateTaskListener() {
+                    @Override
+                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                        throw new IllegalStateException(source);
+                    }
+
+                    @Override
+                    public void onFailure(String source, Throwable t) {
+                    }
+                }
+        );
+
+        latch.await();
+        assertTrue(published.get());
+    }
+
+    // test that for a single thread, tasks are executed in the order
+    // that they are submitted
+    public void testClusterStateUpdateTasksAreExecutedInOrder() throws BrokenBarrierException, InterruptedException {
+        class TaskExecutor implements ClusterStateTaskExecutor<Integer> {
+            List<Integer> tasks = new ArrayList<>();
+
+            @Override
+            public BatchResult<Integer> execute(ClusterState currentState, List<Integer> tasks) throws Exception {
+                this.tasks.addAll(tasks);
+                return BatchResult.<Integer>builder().successes(tasks).build(ClusterState.builder(currentState).build());
+            }
+
+            @Override
+            public boolean runOnlyOnMaster() {
+                return false;
+            }
+        }
+
+        int numberOfThreads = randomIntBetween(2, 8);
+        TaskExecutor[] executors = new TaskExecutor[numberOfThreads];
+        for (int i = 0; i < numberOfThreads; i++) {
+            executors[i] = new TaskExecutor();
+        }
+
+        int tasksSubmittedPerThread = randomIntBetween(2, 1024);
+
+        CopyOnWriteArrayList<Tuple<String, Throwable>> failures = new CopyOnWriteArrayList<>();
+        CountDownLatch updateLatch = new CountDownLatch(numberOfThreads * tasksSubmittedPerThread);
+
+        ClusterStateTaskListener listener = new ClusterStateTaskListener() {
+            @Override
+            public void onFailure(String source, Throwable t) {
+                logger.error("unexpected failure: [{}]", t, source);
+                failures.add(new Tuple<>(source, t));
+                updateLatch.countDown();
+            }
+
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                updateLatch.countDown();
+            }
+        };
+
+        CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads);
+
+        for (int i = 0; i < numberOfThreads; i++) {
+            final int index = i;
+            Thread thread = new Thread(() -> {
+                try {
+                    barrier.await();
+                    for (int j = 0; j < tasksSubmittedPerThread; j++) {
+                        clusterService.submitStateUpdateTask("[" + index + "][" + j + "]", j,
+                                ClusterStateTaskConfig.build(randomFrom(Priority.values())), executors[index], listener);
+                    }
+                    barrier.await();
+                } catch (InterruptedException | BrokenBarrierException e) {
+                    throw new AssertionError(e);
+                }
+            });
+            thread.start();
+        }
+
+        // wait for all threads to be ready
+        barrier.await();
+        // wait for all threads to finish
+        barrier.await();
+
+        updateLatch.await();
+
+        assertThat(failures, empty());
+
+        for (int i = 0; i < numberOfThreads; i++) {
+            assertEquals(tasksSubmittedPerThread, executors[i].tasks.size());
+            for (int j = 0; j < tasksSubmittedPerThread; j++) {
+                assertNotNull(executors[i].tasks.get(j));
+                assertEquals("cluster state update task executed out of order", j, (int) executors[i].tasks.get(j));
+            }
+        }
+    }
+
+    public void testClusterStateBatchedUpdates() throws BrokenBarrierException, InterruptedException {
+        AtomicInteger counter = new AtomicInteger();
+        class Task {
+            private AtomicBoolean state = new AtomicBoolean();
+
+            public void execute() {
+                if (!state.compareAndSet(false, true)) {
+                    throw new IllegalStateException();
+                } else {
+                    counter.incrementAndGet();
+                }
+            }
+        }
+
+        int numberOfThreads = randomIntBetween(2, 8);
+        int tasksSubmittedPerThread = randomIntBetween(1, 1024);
+        int numberOfExecutors = Math.max(1, numberOfThreads / 4);
+        final Semaphore semaphore = new Semaphore(numberOfExecutors);
+
+        class TaskExecutor implements ClusterStateTaskExecutor<Task> {
+            private AtomicInteger counter = new AtomicInteger();
+            private AtomicInteger batches = new AtomicInteger();
+            private AtomicInteger published = new AtomicInteger();
+
+            @Override
+            public BatchResult<Task> execute(ClusterState currentState, List<Task> tasks) throws Exception {
+                tasks.forEach(task -> task.execute());
+                counter.addAndGet(tasks.size());
+                ClusterState maybeUpdatedClusterState = currentState;
+                if (randomBoolean()) {
+                    maybeUpdatedClusterState = ClusterState.builder(currentState).build();
+                    batches.incrementAndGet();
+                    semaphore.acquire();
+                }
+                return BatchResult.<Task>builder().successes(tasks).build(maybeUpdatedClusterState);
+            }
+
+            @Override
+            public boolean runOnlyOnMaster() {
+                return false;
+            }
+
+            @Override
+            public void clusterStatePublished(ClusterState newClusterState) {
+                published.incrementAndGet();
+                semaphore.release();
+            }
+        }
+
+        ConcurrentMap<String, AtomicInteger> counters = new ConcurrentHashMap<>();
+        CountDownLatch updateLatch = new CountDownLatch(numberOfThreads * tasksSubmittedPerThread);
+        ClusterStateTaskListener listener = new ClusterStateTaskListener() {
+            @Override
+            public void onFailure(String source, Throwable t) {
+                assert false;
+            }
+
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                counters.computeIfAbsent(source, key -> new AtomicInteger()).incrementAndGet();
+                updateLatch.countDown();
+            }
+        };
+
+        List<TaskExecutor> executors = new ArrayList<>();
+        for (int i = 0; i < numberOfExecutors; i++) {
+            executors.add(new TaskExecutor());
+        }
+
+        // randomly assign tasks to executors
+        List<TaskExecutor> assignments = new ArrayList<>();
+        for (int i = 0; i < numberOfThreads; i++) {
+            for (int j = 0; j < tasksSubmittedPerThread; j++) {
+                assignments.add(randomFrom(executors));
+            }
+        }
+
+        Map<TaskExecutor, Integer> counts = new HashMap<>();
+        for (TaskExecutor executor : assignments) {
+            counts.merge(executor, 1, (previous, one) -> previous + one);
+        }
+
+        CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads);
+        for (int i = 0; i < numberOfThreads; i++) {
+            final int index = i;
+            Thread thread = new Thread(() -> {
+                try {
+                    barrier.await();
+                    for (int j = 0; j < tasksSubmittedPerThread; j++) {
+                        ClusterStateTaskExecutor<Task> executor = assignments.get(index * tasksSubmittedPerThread + j);
+                        clusterService.submitStateUpdateTask(
+                                Thread.currentThread().getName(),
+                                new Task(),
+                                ClusterStateTaskConfig.build(randomFrom(Priority.values())),
+                                executor,
+                                listener);
+                    }
+                    barrier.await();
+                } catch (BrokenBarrierException | InterruptedException e) {
+                    throw new AssertionError(e);
+                }
+            });
+            thread.start();
+        }
+
+        // wait for all threads to be ready
+        barrier.await();
+        // wait for all threads to finish
+        barrier.await();
+
+        // wait until all the cluster state updates have been processed
+        updateLatch.await();
+        // and until all of the publication callbacks have completed
+        semaphore.acquire(numberOfExecutors);
+
+        // assert the number of executed tasks is correct
+        assertEquals(numberOfThreads * tasksSubmittedPerThread, counter.get());
+
+        // assert each executor executed the correct number of tasks
+        for (TaskExecutor executor : executors) {
+            if (counts.containsKey(executor)) {
+                assertEquals((int) counts.get(executor), executor.counter.get());
+                assertEquals(executor.batches.get(), executor.published.get());
+            }
+        }
+
+        // assert the correct number of clusterStateProcessed events were triggered
+        for (Map.Entry<String, AtomicInteger> entry : counters.entrySet()) {
+            assertEquals(entry.getValue().get(), tasksSubmittedPerThread);
+        }
+    }
+
+    /**
+     * Note, this test can only work as long as we have a single thread executor executing the state update tasks!
+     */
+    public void testPrioritizedTasks() throws Exception {
+        Settings settings = settingsBuilder()
+                .put("discovery.type", "local")
+                .build();
+        BlockingTask block = new BlockingTask(Priority.IMMEDIATE);
+        clusterService.submitStateUpdateTask("test", block);
+        int taskCount = randomIntBetween(5, 20);
+        Priority[] priorities = Priority.values();
+
+        // will hold all the tasks in the order in which they were executed
+        List<PrioritizedTask> tasks = new ArrayList<>(taskCount);
+        CountDownLatch latch = new CountDownLatch(taskCount);
+        for (int i = 0; i < taskCount; i++) {
+            Priority priority = priorities[randomIntBetween(0, priorities.length - 1)];
+            clusterService.submitStateUpdateTask("test", new PrioritizedTask(priority, latch, tasks));
+        }
+
+        block.release();
+        latch.await();
+
+        Priority prevPriority = null;
+        for (PrioritizedTask task : tasks) {
+            if (prevPriority == null) {
+                prevPriority = task.priority();
+            } else {
+                assertThat(task.priority().sameOrAfter(prevPriority), is(true));
+            }
+        }
+    }
+
+    @TestLogging("cluster:TRACE") // To ensure that we log cluster state events on TRACE level
+    public void testClusterStateUpdateLogging() throws Exception {
+        MockLogAppender mockAppender = new MockLogAppender();
+        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test1", "cluster.service", Level.DEBUG,
+                "*processing [test1]: took [1s] no change in cluster_state"));
+        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test2", "cluster.service", Level.TRACE,
+                "*failed to execute cluster state update in [2s]*"));
+        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test3", "cluster.service", Level.DEBUG,
+                "*processing [test3]: took [3s] done applying updated cluster_state (version: *, uuid: *)"));
+
+        Logger rootLogger = Logger.getRootLogger();
+        rootLogger.addAppender(mockAppender);
+        try {
+            final CountDownLatch latch = new CountDownLatch(4);
+            clusterService.currentTimeOverride = System.nanoTime();
+            clusterService.submitStateUpdateTask("test1", new ClusterStateUpdateTask() {
+                @Override
+                public ClusterState execute(ClusterState currentState) throws Exception {
+                    clusterService.currentTimeOverride += TimeValue.timeValueSeconds(1).nanos();
+                    return currentState;
+                }
+
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(String source, Throwable t) {
+                    fail();
+                }
+            });
+            clusterService.submitStateUpdateTask("test2", new ClusterStateUpdateTask() {
+                @Override
+                public ClusterState execute(ClusterState currentState) {
+                    clusterService.currentTimeOverride += TimeValue.timeValueSeconds(2).nanos();
+                    throw new IllegalArgumentException("Testing handling of exceptions in the cluster state task");
+                }
+
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    fail();
+                }
+
+                @Override
+                public void onFailure(String source, Throwable t) {
+                    latch.countDown();
+                }
+            });
+            clusterService.submitStateUpdateTask("test3", new ClusterStateUpdateTask() {
+                @Override
+                public ClusterState execute(ClusterState currentState) {
+                    clusterService.currentTimeOverride += TimeValue.timeValueSeconds(3).nanos();
+                    return ClusterState.builder(currentState).incrementVersion().build();
+                }
+
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(String source, Throwable t) {
+                    fail();
+                }
+            });
+            // Additional update task to make sure all previous logging made it to the logger
+            // We don't check logging for this on since there is no guarantee that it will occur before our check
+            clusterService.submitStateUpdateTask("test4", new ClusterStateUpdateTask() {
+                @Override
+                public ClusterState execute(ClusterState currentState) {
+                    return currentState;
+                }
+
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(String source, Throwable t) {
+                    fail();
+                }
+            });
+            latch.await();
+        } finally {
+            rootLogger.removeAppender(mockAppender);
+        }
+        mockAppender.assertAllExpectationsMatched();
+    }
+
+    @TestLogging("cluster:WARN") // To ensure that we log cluster state events on WARN level
+    public void testLongClusterStateUpdateLogging() throws Exception {
+        MockLogAppender mockAppender = new MockLogAppender();
+        mockAppender.addExpectation(new MockLogAppender.UnseenEventExpectation("test1 shouldn't see because setting is too low",
+                "cluster.service", Level.WARN, "*cluster state update task [test1] took [*] above the warn threshold of *"));
+        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test2", "cluster.service", Level.WARN,
+                "*cluster state update task [test2] took [32s] above the warn threshold of *"));
+        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test3", "cluster.service", Level.WARN,
+                "*cluster state update task [test3] took [33s] above the warn threshold of *"));
+        mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test4", "cluster.service", Level.WARN,
+                "*cluster state update task [test4] took [34s] above the warn threshold of *"));
+
+        Logger rootLogger = Logger.getRootLogger();
+        rootLogger.addAppender(mockAppender);
+        try {
+            final CountDownLatch latch = new CountDownLatch(5);
+            final CountDownLatch processedFirstTask = new CountDownLatch(1);
+            clusterService.currentTimeOverride = System.nanoTime();
+            clusterService.submitStateUpdateTask("test1", new ClusterStateUpdateTask() {
+                @Override
+                public ClusterState execute(ClusterState currentState) throws Exception {
+                    clusterService.currentTimeOverride += TimeValue.timeValueSeconds(1).nanos();
+                    return currentState;
+                }
+
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    latch.countDown();
+                    processedFirstTask.countDown();
+                }
+
+                @Override
+                public void onFailure(String source, Throwable t) {
+                    fail();
+                }
+            });
+
+            processedFirstTask.await();
+            clusterService.submitStateUpdateTask("test2", new ClusterStateUpdateTask() {
+                @Override
+                public ClusterState execute(ClusterState currentState) throws Exception {
+                    clusterService.currentTimeOverride += TimeValue.timeValueSeconds(32).nanos();
+                    throw new IllegalArgumentException("Testing handling of exceptions in the cluster state task");
+                }
+
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    fail();
+                }
+
+                @Override
+                public void onFailure(String source, Throwable t) {
+                    latch.countDown();
+                }
+            });
+            clusterService.submitStateUpdateTask("test3", new ClusterStateUpdateTask() {
+                @Override
+                public ClusterState execute(ClusterState currentState) throws Exception {
+                    clusterService.currentTimeOverride += TimeValue.timeValueSeconds(33).nanos();
+                    return ClusterState.builder(currentState).incrementVersion().build();
+                }
+
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(String source, Throwable t) {
+                    fail();
+                }
+            });
+            clusterService.submitStateUpdateTask("test4", new ClusterStateUpdateTask() {
+                @Override
+                public ClusterState execute(ClusterState currentState) throws Exception {
+                    clusterService.currentTimeOverride += TimeValue.timeValueSeconds(34).nanos();
+                    return currentState;
+                }
+
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(String source, Throwable t) {
+                    fail();
+                }
+            });
+            // Additional update task to make sure all previous logging made it to the logger
+            // We don't check logging for this on since there is no guarantee that it will occur before our check
+            clusterService.submitStateUpdateTask("test5", new ClusterStateUpdateTask() {
+                @Override
+                public ClusterState execute(ClusterState currentState) {
+                    return currentState;
+                }
+
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(String source, Throwable t) {
+                    fail();
+                }
+            });
+            latch.await();
+        } finally {
+            rootLogger.removeAppender(mockAppender);
+        }
+        mockAppender.assertAllExpectationsMatched();
+    }
+
+    private static class BlockingTask extends ClusterStateUpdateTask {
+        private final CountDownLatch latch = new CountDownLatch(1);
+
+        public BlockingTask(Priority priority) {
+            super(priority);
+        }
+
+        @Override
+        public ClusterState execute(ClusterState currentState) throws Exception {
+            latch.await();
+            return currentState;
+        }
+
+        @Override
+        public void onFailure(String source, Throwable t) {
+        }
+
+        public void release() {
+            latch.countDown();
+        }
+
+    }
+
+    private static class PrioritizedTask extends ClusterStateUpdateTask {
+
+        private final CountDownLatch latch;
+        private final List<PrioritizedTask> tasks;
+
+        private PrioritizedTask(Priority priority, CountDownLatch latch, List<PrioritizedTask> tasks) {
+            super(priority);
+            this.latch = latch;
+            this.tasks = tasks;
+        }
+
+        @Override
+        public ClusterState execute(ClusterState currentState) throws Exception {
+            tasks.add(this);
+            latch.countDown();
+            return currentState;
+        }
+
+        @Override
+        public void onFailure(String source, Throwable t) {
+            latch.countDown();
+        }
+    }
+
+    static class TimedClusterService extends ClusterService {
+
+        public volatile Long currentTimeOverride = null;
+
+        public TimedClusterService(Settings settings, OperationRouting operationRouting, ClusterSettings clusterSettings,
+                                   ThreadPool threadPool, ClusterName clusterName) {
+            super(settings, operationRouting, clusterSettings, threadPool, clusterName);
+        }
+
+        @Override
+        protected long currentTimeInNanos() {
+            if (currentTimeOverride != null) {
+                return currentTimeOverride;
+            }
+            return super.currentTimeInNanos();
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceUtils.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceUtils.java
new file mode 100644
index 00000000000..28b921c82da
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceUtils.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.service;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.NodeConnectionsService;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.concurrent.CountDownLatch;
+
+import static junit.framework.TestCase.fail;
+
+public class ClusterServiceUtils {
+
+    public static ClusterService createClusterService(ThreadPool threadPool) {
+        ClusterService clusterService = new ClusterService(Settings.EMPTY, null,
+                new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                threadPool, new ClusterName("ClusterServiceTests"));
+        clusterService.setLocalNode(new DiscoveryNode("node", DummyTransportAddress.INSTANCE, Version.CURRENT));
+        clusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) {
+            @Override
+            public void connectToAddedNodes(ClusterChangedEvent event) {
+                // skip
+            }
+
+            @Override
+            public void disconnectFromRemovedNodes(ClusterChangedEvent event) {
+                // skip
+            }
+        });
+        clusterService.setClusterStatePublisher((event, ackListener) -> {
+        });
+        clusterService.start();
+        return clusterService;
+    }
+
+    public static ClusterService createClusterService(ClusterState initialState, ThreadPool threadPool) {
+        ClusterService clusterService = createClusterService(threadPool);
+        setState(clusterService, initialState);
+        return clusterService;
+    }
+
+    public static void setState(ClusterService clusterService, ClusterState.Builder clusterStateBuilder) {
+        setState(clusterService, clusterStateBuilder.build());
+    }
+
+    public static void setState(ClusterService clusterService, ClusterState clusterState) {
+        CountDownLatch latch = new CountDownLatch(1);
+        clusterService.submitStateUpdateTask("test setting state", new ClusterStateUpdateTask() {
+            @Override
+            public ClusterState execute(ClusterState currentState) throws Exception {
+                // make sure we increment versions as listener may depend on it for change
+                return ClusterState.builder(clusterState).version(currentState.version() + 1).build();
+            }
+
+            @Override
+            public boolean runOnlyOnMaster() {
+                return false;
+            }
+
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                latch.countDown();
+            }
+
+            @Override
+            public void onFailure(String source, Throwable t) {
+                fail("unexpected exception" + t);
+            }
+        });
+        try {
+            latch.await();
+        } catch (InterruptedException e) {
+            throw new ElasticsearchException("unexpected interruption", e);
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java b/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java
index ed03c918c31..e61bbc5f719 100644
--- a/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java
@@ -22,19 +22,15 @@ package org.elasticsearch.cluster.settings;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 
 import java.util.Collection;
-import java.util.Collections;
 
 import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -50,6 +46,11 @@ public class SettingsFilteringIT extends ESIntegTestCase {
     }
 
     public static class SettingsFilteringPlugin extends Plugin {
+        public static final Setting<Boolean> SOME_NODE_SETTING =
+            Setting.boolSetting("some.node.setting", false, Property.NodeScope, Property.Filtered);
+        public static final Setting<Boolean> SOME_OTHER_NODE_SETTING =
+            Setting.boolSetting("some.other.node.setting", false, Property.NodeScope);
+
         /**
          * The name of the plugin.
          */
@@ -72,10 +73,9 @@ public class SettingsFilteringIT extends ESIntegTestCase {
         }
 
         public void onModule(SettingsModule module) {
-            module.registerSetting(Setting.groupSetting("index.filter_test.", false, Setting.Scope.INDEX));
-            module.registerSetting(Setting.boolSetting("some.node.setting", false,  false, Setting.Scope.CLUSTER));
-            module.registerSetting(Setting.boolSetting("some.other.node.setting", false,  false, Setting.Scope.CLUSTER));
-            module.registerSettingsFilter("some.node.setting");
+            module.registerSetting(SOME_NODE_SETTING);
+            module.registerSetting(SOME_OTHER_NODE_SETTING);
+            module.registerSetting(Setting.groupSetting("index.filter_test.", Property.IndexScope));
             module.registerSettingsFilter("index.filter_test.foo");
             module.registerSettingsFilter("index.filter_test.bar*");
         }
@@ -104,8 +104,8 @@ public class SettingsFilteringIT extends ESIntegTestCase {
         for(NodeInfo info : nodeInfos.getNodes()) {
             Settings settings = info.getSettings();
             assertNotNull(settings);
-            assertNull(settings.get("some.node.setting"));
-            assertTrue(settings.getAsBoolean("some.other.node.setting", false));
+            assertNull(settings.get(SettingsFilteringPlugin.SOME_NODE_SETTING.getKey()));
+            assertTrue(settings.getAsBoolean(SettingsFilteringPlugin.SOME_OTHER_NODE_SETTING.getKey(), false));
             assertEquals(settings.get("node.name"), info.getNode().getName());
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
index 95a65f82924..7b34685da7d 100644
--- a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
+++ b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
@@ -154,7 +154,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
     }
 
     public void testStreamInputBulkReadWithOffset() throws IOException {
-        int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20));
+        final int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20));
         BytesReference pbr = getRandomizedPagedBytesReference(length);
         StreamInput si = pbr.streamInput();
         assertNotNull(si);
@@ -162,6 +162,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
         // read a bunch of single bytes one by one
         int offset = randomIntBetween(1, length / 2);
         for (int i = 0; i < offset; i++) {
+            assertEquals(si.available(), length - i);
             assertEquals(pbr.get(i), si.readByte());
         }
 
@@ -176,6 +177,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
         // bulk-read all
         si.readFully(targetBytes);
         assertArrayEquals(pbrBytesWithOffset, targetBytes);
+        assertEquals(si.available(), 0);
     }
 
     public void testRandomReads() throws IOException {
@@ -216,18 +218,22 @@ public class PagedBytesReferenceTests extends ESTestCase {
         int sliceLength = length - sliceOffset;
         BytesReference slice = pbr.slice(sliceOffset, sliceLength);
         StreamInput sliceInput = slice.streamInput();
+        assertEquals(sliceInput.available(), sliceLength);
 
         // single reads
         assertEquals(slice.get(0), sliceInput.readByte());
         assertEquals(slice.get(1), sliceInput.readByte());
         assertEquals(slice.get(2), sliceInput.readByte());
+        assertEquals(sliceInput.available(), sliceLength - 3);
 
         // reset the slice stream for bulk reading
         sliceInput.reset();
+        assertEquals(sliceInput.available(), sliceLength);
 
         // bulk read
         byte[] sliceBytes = new byte[sliceLength];
         sliceInput.readFully(sliceBytes);
+        assertEquals(sliceInput.available(), 0);
 
         // compare slice content with upper half of original
         byte[] pbrSliceBytes = Arrays.copyOfRange(pbr.toBytes(), sliceOffset, length);
@@ -239,11 +245,14 @@ public class PagedBytesReferenceTests extends ESTestCase {
         assertArrayEquals(sliceBytes, sliceToBytes);
 
         sliceInput.reset();
+        assertEquals(sliceInput.available(), sliceLength);
         byte[] buffer = new byte[sliceLength + scaledRandomIntBetween(1, 100)];
         int offset = scaledRandomIntBetween(0, Math.max(1, buffer.length - sliceLength - 1));
         int read = sliceInput.read(buffer, offset, sliceLength / 2);
+        assertEquals(sliceInput.available(), sliceLength - read);
         sliceInput.read(buffer, offset + read, sliceLength);
         assertArrayEquals(sliceBytes, Arrays.copyOfRange(buffer, offset, offset + sliceLength));
+        assertEquals(sliceInput.available(), 0);
     }
 
     public void testWriteToOutputStream() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/common/cache/CacheTests.java b/core/src/test/java/org/elasticsearch/common/cache/CacheTests.java
index 921c66f7acb..3b88a3bdcfe 100644
--- a/core/src/test/java/org/elasticsearch/common/cache/CacheTests.java
+++ b/core/src/test/java/org/elasticsearch/common/cache/CacheTests.java
@@ -55,7 +55,7 @@ public class CacheTests extends ESTestCase {
     public void setUp() throws Exception {
         super.setUp();
         numberOfEntries = randomIntBetween(1000, 10000);
-        logger.debug("numberOfEntries: " + numberOfEntries);
+        logger.debug("numberOfEntries: {}", numberOfEntries);
     }
 
     // cache some entries, then randomly lookup keys that do not exist, then check the stats
diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java
index 9311db44da0..63f6ecd0e67 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java
@@ -50,15 +50,7 @@ public abstract class AbstractShapeBuilderTestCase<SB extends ShapeBuilder> exte
     public static void init() {
         if (namedWriteableRegistry == null) {
             namedWriteableRegistry = new NamedWriteableRegistry();
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, PointBuilder.PROTOTYPE);
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, CircleBuilder.PROTOTYPE);
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, EnvelopeBuilder.PROTOTYPE);
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, MultiPointBuilder.PROTOTYPE);
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, LineStringBuilder.PROTOTYPE);
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, MultiLineStringBuilder.PROTOTYPE);
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, PolygonBuilder.PROTOTYPE);
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, MultiPolygonBuilder.PROTOTYPE);
-            namedWriteableRegistry.registerPrototype(ShapeBuilder.class, GeometryCollectionBuilder.PROTOTYPE);
+            ShapeBuilders.register(namedWriteableRegistry);
         }
     }
 
@@ -146,8 +138,7 @@ public abstract class AbstractShapeBuilderTestCase<SB extends ShapeBuilder> exte
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             original.writeTo(output);
             try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
-                ShapeBuilder prototype = (ShapeBuilder) namedWriteableRegistry.getPrototype(ShapeBuilder.class, original.getWriteableName());
-                return prototype.readFrom(in);
+                return namedWriteableRegistry.getReader(ShapeBuilder.class, original.getWriteableName()).read(in);
             }
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/EnvelopeBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/builders/EnvelopeBuilderTests.java
index 881db868ef9..c2730f91df6 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/builders/EnvelopeBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/builders/EnvelopeBuilderTests.java
@@ -19,30 +19,21 @@
 
 package org.elasticsearch.common.geo.builders;
 
-import org.locationtech.spatial4j.shape.Rectangle;
 import com.vividsolutions.jts.geom.Coordinate;
+
 import org.elasticsearch.test.geo.RandomShapeGenerator;
+import org.locationtech.spatial4j.shape.Rectangle;
 
 import java.io.IOException;
 
-import static org.hamcrest.Matchers.equalTo;
-
 public class EnvelopeBuilderTests extends AbstractShapeBuilderTestCase<EnvelopeBuilder> {
 
     public void testInvalidConstructorArgs() {
-        try {
-            new EnvelopeBuilder(null, new Coordinate(1.0, -1.0));
-            fail("Exception expected");
-        } catch (NullPointerException e) {
-            assertThat("topLeft of envelope cannot be null", equalTo(e.getMessage()));
-        }
-
-        try {
-            new EnvelopeBuilder(new Coordinate(1.0, -1.0), null);
-            fail("Exception expected");
-        } catch (NullPointerException e) {
-            assertThat("bottomRight of envelope cannot be null", equalTo(e.getMessage()));
-        }
+        NullPointerException e;
+        e = expectThrows(NullPointerException.class, () -> new EnvelopeBuilder(null, new Coordinate(1.0, -1.0)));
+        assertEquals("topLeft of envelope cannot be null", e.getMessage());
+        e = expectThrows(NullPointerException.class, () -> new EnvelopeBuilder(new Coordinate(1.0, -1.0), null));
+        assertEquals("bottomRight of envelope cannot be null", e.getMessage());
     }
 
     @Override
@@ -60,16 +51,21 @@ public class EnvelopeBuilderTests extends AbstractShapeBuilderTestCase<EnvelopeB
         // move one corner to the middle of original
         switch (randomIntBetween(0, 3)) {
         case 0:
-            mutation = new EnvelopeBuilder(new Coordinate(randomDoubleBetween(-180.0, original.bottomRight().x, true), original.topLeft().y), original.bottomRight());
+            mutation = new EnvelopeBuilder(
+                    new Coordinate(randomDoubleBetween(-180.0, original.bottomRight().x, true), original.topLeft().y),
+                    original.bottomRight());
             break;
         case 1:
-            mutation = new EnvelopeBuilder(new Coordinate(original.topLeft().x, randomDoubleBetween(original.bottomRight().y, 90.0, true)), original.bottomRight());
+            mutation = new EnvelopeBuilder(new Coordinate(original.topLeft().x, randomDoubleBetween(original.bottomRight().y, 90.0, true)),
+                    original.bottomRight());
             break;
         case 2:
-            mutation = new EnvelopeBuilder(original.topLeft(), new Coordinate(randomDoubleBetween(original.topLeft().x, 180.0, true), original.bottomRight().y));
+            mutation = new EnvelopeBuilder(original.topLeft(),
+                    new Coordinate(randomDoubleBetween(original.topLeft().x, 180.0, true), original.bottomRight().y));
             break;
         case 3:
-            mutation = new EnvelopeBuilder(original.topLeft(), new Coordinate(original.bottomRight().x, randomDoubleBetween(-90.0, original.topLeft().y, true)));
+            mutation = new EnvelopeBuilder(original.topLeft(),
+                    new Coordinate(original.bottomRight().x, randomDoubleBetween(-90.0, original.topLeft().y, true)));
             break;
         }
         return mutation;
diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/LineStringBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/builders/LineStringBuilderTests.java
index f6fcf8449d3..e96c35287cf 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/builders/LineStringBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/builders/LineStringBuilderTests.java
@@ -27,31 +27,15 @@ import org.elasticsearch.test.geo.RandomShapeGenerator.ShapeType;
 import java.io.IOException;
 import java.util.List;
 
-import static org.hamcrest.Matchers.equalTo;
-
 public class LineStringBuilderTests extends AbstractShapeBuilderTestCase<LineStringBuilder> {
 
     public void testInvalidConstructorArgs() {
-        try {
-            new LineStringBuilder((List<Coordinate>) null);
-            fail("Exception expected");
-        } catch (IllegalArgumentException e) {
-            assertThat("cannot create point collection with empty set of points", equalTo(e.getMessage()));
-        }
-
-       try {
-            new LineStringBuilder(new CoordinatesBuilder());
-            fail("Exception expected");
-        } catch (IllegalArgumentException e) {
-            assertThat("cannot create point collection with empty set of points", equalTo(e.getMessage()));
-        }
-
-       try {
-           new LineStringBuilder(new CoordinatesBuilder().coordinate(0.0, 0.0));
-           fail("Exception expected");
-       } catch (IllegalArgumentException e) {
-           assertThat("invalid number of points in LineString (found [1] - must be >= 2)", equalTo(e.getMessage()));
-       }
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new LineStringBuilder((List<Coordinate>) null));
+        assertEquals("cannot create point collection with empty set of points", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class, () -> new LineStringBuilder(new CoordinatesBuilder()));
+        assertEquals("cannot create point collection with empty set of points", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class, () -> new LineStringBuilder(new CoordinatesBuilder().coordinate(0.0, 0.0)));
+        assertEquals("invalid number of points in LineString (found [1] - must be >= 2)", e.getMessage());
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilderTests.java
index 3c618fd3696..925d177c573 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilderTests.java
@@ -68,9 +68,6 @@ public class MultiLineStringBuilderTests extends AbstractShapeBuilderTestCase<Mu
     }
 
     static MultiLineStringBuilder createRandomShape() {
-        if (true) {
-            return new MultiLineStringBuilder();
-        }
-        return (MultiLineStringBuilder) RandomShapeGenerator.createShape(getRandom(), ShapeType.MULTILINESTRING);
+        return new MultiLineStringBuilder();
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/MultiPointBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/builders/MultiPointBuilderTests.java
index 006064578e0..ec2eb50bd38 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/builders/MultiPointBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/builders/MultiPointBuilderTests.java
@@ -20,29 +20,20 @@
 package org.elasticsearch.common.geo.builders;
 
 import com.vividsolutions.jts.geom.Coordinate;
+
 import org.elasticsearch.test.geo.RandomShapeGenerator;
 import org.elasticsearch.test.geo.RandomShapeGenerator.ShapeType;
 
 import java.io.IOException;
-
-import static org.hamcrest.Matchers.equalTo;
+import java.util.List;
 
 public class MultiPointBuilderTests extends AbstractShapeBuilderTestCase<MultiPointBuilder> {
 
     public void testInvalidBuilderException() {
-        try {
-            new MultiPointBuilder(null);
-            fail("IllegalArgumentException expected");
-        } catch (IllegalArgumentException e) {
-            assertThat("cannot create point collection with empty set of points", equalTo(e.getMessage()));
-        }
-
-        try {
-            new MultiPointBuilder(new CoordinatesBuilder().build());
-            fail("IllegalArgumentException expected");
-        } catch (IllegalArgumentException e) {
-            assertThat("cannot create point collection with empty set of points", equalTo(e.getMessage()));
-        }
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new MultiPointBuilder((List<Coordinate>) null));
+        assertEquals("cannot create point collection with empty set of points", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class, () -> new MultiPointBuilder(new CoordinatesBuilder().build()));
+        assertEquals("cannot create point collection with empty set of points", e.getMessage());
 
         // one point is minimum
         new MultiPointBuilder(new CoordinatesBuilder().coordinate(0.0, 0.0).build());
diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java
index 24e0bc85710..9a35690f947 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java
@@ -80,7 +80,8 @@ public class PolygonBuilderTests extends AbstractShapeBuilderTestCase<PolygonBui
      * This is done so we don't have to expose a setter for orientation in the actual class
      */
     private static PolygonBuilder polyWithOposingOrientation(PolygonBuilder pb) {
-        PolygonBuilder mutation = new PolygonBuilder(pb.shell(), pb.orientation() == Orientation.LEFT ? Orientation.RIGHT : Orientation.LEFT);
+        PolygonBuilder mutation = new PolygonBuilder(pb.shell(),
+                pb.orientation() == Orientation.LEFT ? Orientation.RIGHT : Orientation.LEFT);
         for (LineStringBuilder hole : pb.holes()) {
             mutation.hole(hole);
         }
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/AbstractWriteableEnumTestCase.java b/core/src/test/java/org/elasticsearch/common/io/stream/AbstractWriteableEnumTestCase.java
new file mode 100644
index 00000000000..b8be6fb1493
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/AbstractWriteableEnumTestCase.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.io.stream;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * Abstract class offering base functionality for testing @{link Writeable} enums.
+ */
+public abstract class AbstractWriteableEnumTestCase extends ESTestCase {
+
+    /**
+     * Test that the ordinals for the enum are consistent (i.e. the order hasn't changed)
+     * because writing an enum to a stream often uses the ordinal value.
+     */
+    public abstract void testValidOrdinals();
+
+    /**
+     * Test that the conversion from a string to enum is correct.
+     */
+    public abstract void testFromString();
+
+    /**
+     * Test that the correct enum value is produced from the serialized value in the {@link StreamInput}.
+     */
+    public abstract void testReadFrom() throws IOException;
+
+    /**
+     * Test that the correct serialized value is produced from the {@link StreamOutput}.
+     */
+    public abstract void testWriteTo() throws IOException;
+
+    // a convenience method for testing the write of a writeable enum
+    protected static void assertWriteToStream(final Writeable writeableEnum, final int ordinal) throws IOException {
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            writeableEnum.writeTo(out);
+            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+                assertThat(in.readVInt(), equalTo(ordinal));
+            }
+        }
+    }
+
+    // a convenience method for testing the read of a writeable enum
+    protected static <T extends Writeable<T>> void assertReadFromStream(final int ordinal, final Writeable<T> expected) throws IOException {
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.writeVInt(ordinal);
+            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+                assertThat(expected.readFrom(in), equalTo(expected));
+            }
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
index 7f232363f73..7a2828c0a11 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
@@ -29,8 +29,10 @@ import java.io.IOException;
 import java.util.Objects;
 
 import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.startsWith;
 
 /**
  * Tests for {@link BytesStreamOutput} paging behaviour.
@@ -273,7 +275,9 @@ public class BytesStreamsTests extends ESTestCase {
         out.writeString("hello");
         out.writeString("goodbye");
         out.writeGenericValue(BytesRefs.toBytesRef("bytesref"));
+        final byte[] bytes = out.bytes().toBytes();
         StreamInput in = StreamInput.wrap(out.bytes().toBytes());
+        assertEquals(in.available(), bytes.length);
         assertThat(in.readBoolean(), equalTo(false));
         assertThat(in.readByte(), equalTo((byte)1));
         assertThat(in.readShort(), equalTo((short)-1));
@@ -299,42 +303,38 @@ public class BytesStreamsTests extends ESTestCase {
     public void testNamedWriteable() throws IOException {
         BytesStreamOutput out = new BytesStreamOutput();
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
-        namedWriteableRegistry.registerPrototype(BaseNamedWriteable.class, new TestNamedWriteable(null, null));
+        namedWriteableRegistry.register(BaseNamedWriteable.class, TestNamedWriteable.NAME, TestNamedWriteable::new);
         TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10));
         out.writeNamedWriteable(namedWriteableIn);
-        StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes().toBytes()), namedWriteableRegistry);
+        byte[] bytes = out.bytes().toBytes();
+        StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry);
+        assertEquals(in.available(), bytes.length);
         BaseNamedWriteable namedWriteableOut = in.readNamedWriteable(BaseNamedWriteable.class);
         assertEquals(namedWriteableOut, namedWriteableIn);
+        assertEquals(in.available(), 0);
     }
 
     public void testNamedWriteableDuplicates() throws IOException {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
-        namedWriteableRegistry.registerPrototype(BaseNamedWriteable.class, new TestNamedWriteable(null, null));
-        try {
-            namedWriteableRegistry.registerPrototype(BaseNamedWriteable.class, new TestNamedWriteable(null, null));
-            fail("registerPrototype should have failed");
-        } catch(IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("named writeable of type [" + TestNamedWriteable.class.getName() + "] with name [" + TestNamedWriteable.NAME + "] is already registered by type ["
-                    + TestNamedWriteable.class.getName() + "] within category [" + BaseNamedWriteable.class.getName() + "]"));
-        }
+        namedWriteableRegistry.register(BaseNamedWriteable.class, TestNamedWriteable.NAME, TestNamedWriteable::new);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> namedWriteableRegistry.register(BaseNamedWriteable.class, TestNamedWriteable.NAME, TestNamedWriteable::new));
+        assertThat(e.getMessage(), startsWith("named writeable [" + BaseNamedWriteable.class.getName() + "][" + TestNamedWriteable.NAME
+                + "] is already registered by ["));
     }
 
     public void testNamedWriteableUnknownCategory() throws IOException {
         BytesStreamOutput out = new BytesStreamOutput();
         out.writeNamedWriteable(new TestNamedWriteable("test1", "test2"));
         StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes().toBytes()), new NamedWriteableRegistry());
-        try {
-            //no named writeable registered with given name, can write but cannot read it back
-            in.readNamedWriteable(BaseNamedWriteable.class);
-            fail("read should have failed");
-        } catch(IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("unknown named writeable category [" + BaseNamedWriteable.class.getName() + "]"));
-        }
+        //no named writeable registered with given name, can write but cannot read it back
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> in.readNamedWriteable(BaseNamedWriteable.class));
+        assertThat(e.getMessage(), equalTo("unknown named writeable category [" + BaseNamedWriteable.class.getName() + "]"));
     }
 
     public void testNamedWriteableUnknownNamedWriteable() throws IOException {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
-        namedWriteableRegistry.registerPrototype(BaseNamedWriteable.class, new TestNamedWriteable(null, null));
+        namedWriteableRegistry.register(BaseNamedWriteable.class, TestNamedWriteable.NAME, TestNamedWriteable::new);
         BytesStreamOutput out = new BytesStreamOutput();
         out.writeNamedWriteable(new NamedWriteable() {
             @Override
@@ -357,7 +357,7 @@ public class BytesStreamsTests extends ESTestCase {
             in.readNamedWriteable(BaseNamedWriteable.class);
             fail("read should have failed");
         } catch(IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("unknown named writeable with name [unknown] within category [" + BaseNamedWriteable.class.getName() + "]"));
+            assertThat(e.getMessage(), equalTo("unknown named writeable [" + BaseNamedWriteable.class.getName() + "][unknown]"));
         }
     }
 
@@ -374,6 +374,27 @@ public class BytesStreamsTests extends ESTestCase {
         }
     }
 
+    public void testNamedWriteableReaderReturnsNull() throws IOException {
+        BytesStreamOutput out = new BytesStreamOutput();
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
+        namedWriteableRegistry.register(BaseNamedWriteable.class, TestNamedWriteable.NAME, (StreamInput in) -> null);
+        TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10));
+        out.writeNamedWriteable(namedWriteableIn);
+        byte[] bytes = out.bytes().toBytes();
+        StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry);
+        assertEquals(in.available(), bytes.length);
+        IOException e = expectThrows(IOException.class, () -> in.readNamedWriteable(BaseNamedWriteable.class));
+        assertThat(e.getMessage(), endsWith("] returned null which is not allowed and probably means it screwed up the stream."));
+    }
+
+    public void testOptionalWriteableReaderReturnsNull() throws IOException {
+        BytesStreamOutput out = new BytesStreamOutput();
+        out.writeOptionalWriteable(new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)));
+        StreamInput in = StreamInput.wrap(out.bytes().toBytes());
+        IOException e = expectThrows(IOException.class, () -> in.readOptionalWriteable((StreamInput ignored) -> null));
+        assertThat(e.getMessage(), endsWith("] returned null which is not allowed and probably means it screwed up the stream."));
+    }
+
     private static abstract class BaseNamedWriteable<T> implements NamedWriteable<T> {
 
     }
@@ -390,6 +411,11 @@ public class BytesStreamsTests extends ESTestCase {
             this.field2 = field2;
         }
 
+        public TestNamedWriteable(StreamInput in) throws IOException {
+            field1 = in.readString();
+            field2 = in.readString();
+        }
+
         @Override
         public String getWriteableName() {
             return NAME;
@@ -401,11 +427,6 @@ public class BytesStreamsTests extends ESTestCase {
             out.writeString(field2);
         }
 
-        @Override
-        public TestNamedWriteable readFrom(StreamInput in) throws IOException {
-            return new TestNamedWriteable(in.readString(), in.readString());
-        }
-
         @Override
         public boolean equals(Object o) {
             if (this == o) return true;
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
index b5f26dba8a5..72f933462e0 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
@@ -23,7 +23,10 @@ import org.elasticsearch.common.bytes.ByteBufferBytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.ByteArrayInputStream;
+import java.io.FilterInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -93,4 +96,29 @@ public class StreamTests extends ESTestCase {
             index++;
         }
     }
+
+    public void testFilterStreamInputDelegatesAvailable() throws IOException {
+        final int length = randomIntBetween(1, 1024);
+        StreamInput delegate = StreamInput.wrap(new byte[length]);
+
+        FilterStreamInput filterInputStream = new FilterStreamInput(delegate) {};
+        assertEquals(filterInputStream.available(), length);
+
+        // read some bytes
+        final int bytesToRead = randomIntBetween(1, length);
+        filterInputStream.readBytes(new byte[bytesToRead], 0, bytesToRead);
+        assertEquals(filterInputStream.available(), length - bytesToRead);
+    }
+
+    public void testInputStreamStreamInputDelegatesAvailable() throws IOException {
+        final int length = randomIntBetween(1, 1024);
+        ByteArrayInputStream is = new ByteArrayInputStream(new byte[length]);
+        InputStreamStreamInput streamInput = new InputStreamStreamInput(is);
+        assertEquals(streamInput.available(), length);
+
+        // read some bytes
+        final int bytesToRead = randomIntBetween(1, length);
+        streamInput.readBytes(new byte[bytesToRead], 0, bytesToRead);
+        assertEquals(streamInput.available(), length - bytesToRead);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/logging/LoggingConfigurationTests.java b/core/src/test/java/org/elasticsearch/common/logging/LoggingConfigurationTests.java
index 0cca19d33bf..5c812cca0a7 100644
--- a/core/src/test/java/org/elasticsearch/common/logging/LoggingConfigurationTests.java
+++ b/core/src/test/java/org/elasticsearch/common/logging/LoggingConfigurationTests.java
@@ -27,7 +27,7 @@ import java.util.Arrays;
 
 import org.apache.log4j.Appender;
 import org.apache.log4j.Logger;
-import org.elasticsearch.common.cli.MockTerminal;
+import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
index 4090daa23a0..4d8f3e6e58d 100644
--- a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
@@ -19,12 +19,18 @@
 
 package org.elasticsearch.common.network;
 
+import org.elasticsearch.action.support.replication.ReplicationTask;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.ModuleTestCase;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.http.HttpInfo;
 import org.elasticsearch.http.HttpServerAdapter;
 import org.elasticsearch.http.HttpServerTransport;
@@ -36,10 +42,16 @@ import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.cat.AbstractCatAction;
 import org.elasticsearch.rest.action.cat.RestNodesAction;
 import org.elasticsearch.rest.action.main.RestMainAction;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.Task.Status;
 import org.elasticsearch.test.transport.AssertingLocalTransport;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.sameInstance;
+
 public class NetworkModuleTests extends ModuleTestCase {
 
     static class FakeTransportService extends TransportService {
@@ -104,36 +116,36 @@ public class NetworkModuleTests extends ModuleTestCase {
 
     public void testRegisterTransportService() {
         Settings settings = Settings.builder().put(NetworkModule.TRANSPORT_SERVICE_TYPE_KEY, "custom").build();
-        NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, null);
+        NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, new NamedWriteableRegistry());
         module.registerTransportService("custom", FakeTransportService.class);
         assertBinding(module, TransportService.class, FakeTransportService.class);
 
         // check it works with transport only as well
-        module = new NetworkModule(new NetworkService(settings), settings, true, null);
+        module = new NetworkModule(new NetworkService(settings), settings, true, new NamedWriteableRegistry());
         module.registerTransportService("custom", FakeTransportService.class);
         assertBinding(module, TransportService.class, FakeTransportService.class);
     }
 
     public void testRegisterTransport() {
         Settings settings = Settings.builder().put(NetworkModule.TRANSPORT_TYPE_KEY, "custom").build();
-        NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, null);
+        NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, new NamedWriteableRegistry());
         module.registerTransport("custom", FakeTransport.class);
         assertBinding(module, Transport.class, FakeTransport.class);
 
         // check it works with transport only as well
-        module = new NetworkModule(new NetworkService(settings), settings, true, null);
+        module = new NetworkModule(new NetworkService(settings), settings, true, new NamedWriteableRegistry());
         module.registerTransport("custom", FakeTransport.class);
         assertBinding(module, Transport.class, FakeTransport.class);
     }
 
     public void testRegisterHttpTransport() {
         Settings settings = Settings.builder().put(NetworkModule.HTTP_TYPE_SETTING.getKey(), "custom").build();
-        NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, null);
+        NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, new NamedWriteableRegistry());
         module.registerHttpTransport("custom", FakeHttpTransport.class);
         assertBinding(module, HttpServerTransport.class, FakeHttpTransport.class);
 
         // check registration not allowed for transport only
-        module = new NetworkModule(new NetworkService(settings), settings, true, null);
+        module = new NetworkModule(new NetworkService(settings), settings, true, new NamedWriteableRegistry());
         try {
             module.registerHttpTransport("custom", FakeHttpTransport.class);
             fail();
@@ -144,19 +156,19 @@ public class NetworkModuleTests extends ModuleTestCase {
 
         // not added if http is disabled
         settings = Settings.builder().put(NetworkModule.HTTP_ENABLED.getKey(), false).build();
-        module = new NetworkModule(new NetworkService(settings), settings, false, null);
+        module = new NetworkModule(new NetworkService(settings), settings, false, new NamedWriteableRegistry());
         assertNotBound(module, HttpServerTransport.class);
     }
 
     public void testRegisterRestHandler() {
         Settings settings = Settings.EMPTY;
-        NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, null);
+        NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, new NamedWriteableRegistry());
         module.registerRestHandler(FakeRestHandler.class);
         // also check a builtin is bound
         assertSetMultiBinding(module, RestHandler.class, FakeRestHandler.class, RestMainAction.class);
 
         // check registration not allowed for transport only
-        module = new NetworkModule(new NetworkService(settings), settings, true, null);
+        module = new NetworkModule(new NetworkService(settings), settings, true, new NamedWriteableRegistry());
         try {
             module.registerRestHandler(FakeRestHandler.class);
             fail();
@@ -168,9 +180,45 @@ public class NetworkModuleTests extends ModuleTestCase {
 
     public void testRegisterCatRestHandler() {
         Settings settings = Settings.EMPTY;
-        NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, null);
+        NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, new NamedWriteableRegistry());
         module.registerRestHandler(FakeCatRestHandler.class);
         // also check a builtin is bound
         assertSetMultiBinding(module, AbstractCatAction.class, FakeCatRestHandler.class, RestNodesAction.class);
     }
+
+    public void testRegisterTaskStatus() {
+        NamedWriteableRegistry registry = new NamedWriteableRegistry();
+        Settings settings = Settings.EMPTY;
+        NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, registry);
+
+        // Builtin reader comes back
+        assertNotNull(registry.getReader(Task.Status.class, ReplicationTask.Status.NAME));
+
+        module.registerTaskStatus(DummyTaskStatus.NAME, DummyTaskStatus::new);
+        assertEquals("test", expectThrows(UnsupportedOperationException.class,
+                () -> registry.getReader(Task.Status.class, DummyTaskStatus.NAME).read(null)).getMessage());
+    }
+
+    private class DummyTaskStatus implements Task.Status {
+        public static final String NAME = "dummy";
+
+        public DummyTaskStatus(StreamInput in) {
+            throw new UnsupportedOperationException("test");
+        }
+
+        @Override
+        public String getWriteableName() {
+            return NAME;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java b/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java
index 4d8fbc3add5..405ac566686 100644
--- a/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java
@@ -24,6 +24,7 @@ import org.elasticsearch.test.ESTestCase;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.is;
 
 public class PropertyPlaceholderTests extends ESTestCase {
@@ -33,10 +34,10 @@ public class PropertyPlaceholderTests extends ESTestCase {
         map.put("foo1", "bar1");
         map.put("foo2", "bar2");
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
-        assertEquals("bar1", propertyPlaceholder.replacePlaceholders("{foo1}", placeholderResolver));
-        assertEquals("a bar1b", propertyPlaceholder.replacePlaceholders("a {foo1}b", placeholderResolver));
-        assertEquals("bar1bar2", propertyPlaceholder.replacePlaceholders("{foo1}{foo2}", placeholderResolver));
-        assertEquals("a bar1 b bar2 c", propertyPlaceholder.replacePlaceholders("a {foo1} b {foo2} c", placeholderResolver));
+        assertEquals("bar1", propertyPlaceholder.replacePlaceholders("key", "{foo1}", placeholderResolver));
+        assertEquals("a bar1b", propertyPlaceholder.replacePlaceholders("key", "a {foo1}b", placeholderResolver));
+        assertEquals("bar1bar2", propertyPlaceholder.replacePlaceholders("key", "{foo1}{foo2}", placeholderResolver));
+        assertEquals("a bar1 b bar2 c", propertyPlaceholder.replacePlaceholders("key", "a {foo1} b {foo2} c", placeholderResolver));
     }
 
     public void testVariousPrefixSuffix() {
@@ -47,24 +48,24 @@ public class PropertyPlaceholderTests extends ESTestCase {
         Map<String, String> map = new LinkedHashMap<>();
         map.put("foo", "bar");
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
-        assertEquals("bar", ppEqualsPrefix.replacePlaceholders("{foo}", placeholderResolver));
-        assertEquals("bar", ppLongerPrefix.replacePlaceholders("${foo}", placeholderResolver));
-        assertEquals("bar", ppShorterPrefix.replacePlaceholders("{foo}}", placeholderResolver));
+        assertEquals("bar", ppEqualsPrefix.replacePlaceholders("key", "{foo}", placeholderResolver));
+        assertEquals("bar", ppLongerPrefix.replacePlaceholders("key", "${foo}", placeholderResolver));
+        assertEquals("bar", ppShorterPrefix.replacePlaceholders("key", "{foo}}", placeholderResolver));
     }
 
     public void testDefaultValue() {
         PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false);
         Map<String, String> map = new LinkedHashMap<>();
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
-        assertEquals("bar", propertyPlaceholder.replacePlaceholders("${foo:bar}", placeholderResolver));
-        assertEquals("", propertyPlaceholder.replacePlaceholders("${foo:}", placeholderResolver));
+        assertEquals("bar", propertyPlaceholder.replacePlaceholders("key", "${foo:bar}", placeholderResolver));
+        assertEquals("", propertyPlaceholder.replacePlaceholders("key", "${foo:}", placeholderResolver));
     }
 
     public void testIgnoredUnresolvedPlaceholder() {
         PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", true);
         Map<String, String> map = new LinkedHashMap<>();
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
-        assertEquals("${foo}", propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver));
+        assertEquals("${foo}", propertyPlaceholder.replacePlaceholders("key", "${foo}", placeholderResolver));
     }
 
     public void testNotIgnoredUnresolvedPlaceholder() {
@@ -72,7 +73,7 @@ public class PropertyPlaceholderTests extends ESTestCase {
         Map<String, String> map = new LinkedHashMap<>();
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
         try {
-            propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver);
+            propertyPlaceholder.replacePlaceholders("key", "${foo}", placeholderResolver);
             fail("Expected IllegalArgumentException");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), is("Could not resolve placeholder 'foo'"));
@@ -83,7 +84,7 @@ public class PropertyPlaceholderTests extends ESTestCase {
         PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false);
         Map<String, String> map = new LinkedHashMap<>();
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, true, true);
-        assertEquals("bar", propertyPlaceholder.replacePlaceholders("bar${foo}", placeholderResolver));
+        assertEquals("bar", propertyPlaceholder.replacePlaceholders("key", "bar${foo}", placeholderResolver));
     }
 
     public void testRecursive() {
@@ -93,8 +94,8 @@ public class PropertyPlaceholderTests extends ESTestCase {
         map.put("foo1", "${foo2}");
         map.put("foo2", "bar");
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
-        assertEquals("bar", propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver));
-        assertEquals("abarb", propertyPlaceholder.replacePlaceholders("a${foo}b", placeholderResolver));
+        assertEquals("bar", propertyPlaceholder.replacePlaceholders("key", "${foo}", placeholderResolver));
+        assertEquals("abarb", propertyPlaceholder.replacePlaceholders("key", "a${foo}b", placeholderResolver));
     }
 
     public void testNestedLongerPrefix() {
@@ -105,7 +106,7 @@ public class PropertyPlaceholderTests extends ESTestCase {
         map.put("foo2", "bar");
         map.put("barbar", "baz");
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
-        assertEquals("baz", propertyPlaceholder.replacePlaceholders("${bar${foo}}", placeholderResolver));
+        assertEquals("baz", propertyPlaceholder.replacePlaceholders("key", "${bar${foo}}", placeholderResolver));
     }
 
     public void testNestedSameLengthPrefixSuffix() {
@@ -116,7 +117,7 @@ public class PropertyPlaceholderTests extends ESTestCase {
         map.put("foo2", "bar");
         map.put("barbar", "baz");
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
-        assertEquals("baz", propertyPlaceholder.replacePlaceholders("{bar{foo}}", placeholderResolver));
+        assertEquals("baz", propertyPlaceholder.replacePlaceholders("key", "{bar{foo}}", placeholderResolver));
     }
 
     public void testNestedShorterPrefix() {
@@ -127,7 +128,7 @@ public class PropertyPlaceholderTests extends ESTestCase {
         map.put("foo2", "bar");
         map.put("barbar", "baz");
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
-        assertEquals("baz", propertyPlaceholder.replacePlaceholders("{bar{foo}}}}", placeholderResolver));
+        assertEquals("baz", propertyPlaceholder.replacePlaceholders("key", "{bar{foo}}}}", placeholderResolver));
     }
 
     public void testCircularReference() {
@@ -137,7 +138,7 @@ public class PropertyPlaceholderTests extends ESTestCase {
         map.put("bar", "${foo}");
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
         try {
-            propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver);
+            propertyPlaceholder.replacePlaceholders("key", "${foo}", placeholderResolver);
             fail("Expected IllegalArgumentException");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), is("Circular placeholder reference 'foo' in property definitions"));
@@ -148,7 +149,24 @@ public class PropertyPlaceholderTests extends ESTestCase {
         PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false);
         Map<String, String> map = new LinkedHashMap<>();
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, true, false);
-        assertEquals("bar${foo}", propertyPlaceholder.replacePlaceholders("bar${foo}", placeholderResolver));
+        assertEquals("bar${foo}", propertyPlaceholder.replacePlaceholders("key", "bar${foo}", placeholderResolver));
+    }
+
+    public void testNullKey() {
+        final PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false);
+        final Map<String, String> map = new LinkedHashMap<>();
+        final PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, true, false);
+        expectThrows(NullPointerException.class, () -> propertyPlaceholder.replacePlaceholders(null, "value", placeholderResolver));
+    }
+
+    public void testNullValue() {
+        final PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false);
+        final Map<String, String> map = new LinkedHashMap<>();
+        final PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, true, false);
+        final String key = randomAsciiOfLength(10);
+        NullPointerException e =
+                expectThrows(NullPointerException.class, () -> propertyPlaceholder.replacePlaceholders(key, null, placeholderResolver));
+        assertThat(e, hasToString("java.lang.NullPointerException: value can not be null for [" + key + "]"));
     }
 
     private class SimplePlaceholderResolver implements PropertyPlaceholder.PlaceholderResolver {
diff --git a/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
index 6cc9912924d..d78f0dd033c 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
@@ -23,6 +23,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
 import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.TransportService;
@@ -41,8 +42,8 @@ import java.util.function.Function;
 public class ScopedSettingsTests extends ESTestCase {
 
     public void testAddConsumer() {
-        Setting<Integer> testSetting = Setting.intSetting("foo.bar", 1, true, Setting.Scope.CLUSTER);
-        Setting<Integer> testSetting2 = Setting.intSetting("foo.bar.baz", 1, true, Setting.Scope.CLUSTER);
+        Setting<Integer> testSetting = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope);
+        Setting<Integer> testSetting2 = Setting.intSetting("foo.bar.baz", 1, Property.Dynamic, Property.NodeScope);
         AbstractScopedSettings service = new ClusterSettings(Settings.EMPTY, Collections.singleton(testSetting));
 
         AtomicInteger consumer = new AtomicInteger();
@@ -69,8 +70,8 @@ public class ScopedSettingsTests extends ESTestCase {
     }
 
     public void testApply() {
-        Setting<Integer> testSetting = Setting.intSetting("foo.bar", 1, true, Setting.Scope.CLUSTER);
-        Setting<Integer> testSetting2 = Setting.intSetting("foo.bar.baz", 1, true, Setting.Scope.CLUSTER);
+        Setting<Integer> testSetting = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope);
+        Setting<Integer> testSetting2 = Setting.intSetting("foo.bar.baz", 1, Property.Dynamic, Property.NodeScope);
         AbstractScopedSettings service = new ClusterSettings(Settings.EMPTY, new HashSet<>(Arrays.asList(testSetting, testSetting2)));
 
         AtomicInteger consumer = new AtomicInteger();
@@ -139,7 +140,10 @@ public class ScopedSettingsTests extends ESTestCase {
     }
 
     public void testIsDynamic(){
-        ClusterSettings settings = new ClusterSettings(Settings.EMPTY, new HashSet<>(Arrays.asList(Setting.intSetting("foo.bar", 1, true, Setting.Scope.CLUSTER), Setting.intSetting("foo.bar.baz", 1, false, Setting.Scope.CLUSTER))));
+        ClusterSettings settings =
+            new ClusterSettings(Settings.EMPTY,
+                new HashSet<>(Arrays.asList(Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope),
+                    Setting.intSetting("foo.bar.baz", 1, Property.NodeScope))));
         assertFalse(settings.hasDynamicSetting("foo.bar.baz"));
         assertTrue(settings.hasDynamicSetting("foo.bar"));
         assertNotNull(settings.get("foo.bar.baz"));
@@ -150,8 +154,8 @@ public class ScopedSettingsTests extends ESTestCase {
     }
 
     public void testDiff() throws IOException {
-        Setting<Integer> foobarbaz = Setting.intSetting("foo.bar.baz", 1, false, Setting.Scope.CLUSTER);
-        Setting<Integer> foobar = Setting.intSetting("foo.bar", 1, true, Setting.Scope.CLUSTER);
+        Setting<Integer> foobarbaz = Setting.intSetting("foo.bar.baz", 1, Property.NodeScope);
+        Setting<Integer> foobar = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope);
         ClusterSettings settings = new ClusterSettings(Settings.EMPTY, new HashSet<>(Arrays.asList(foobar, foobarbaz)));
         Settings diff = settings.diff(Settings.builder().put("foo.bar", 5).build(), Settings.EMPTY);
         assertEquals(diff.getAsMap().size(), 1);
@@ -183,6 +187,15 @@ public class ScopedSettingsTests extends ESTestCase {
         assertEquals("boom", copy.get(IndexModule.INDEX_STORE_TYPE_SETTING)); // test fallback to node settings
     }
 
+    public void testValidateWithSuggestion() {
+        IndexScopedSettings settings = new IndexScopedSettings(
+            Settings.EMPTY,
+            IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
+        IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
+            () -> settings.validate(Settings.builder().put("index.numbe_of_replica", "1").build()));
+        assertEquals(iae.getMessage(), "unknown setting [index.numbe_of_replica] did you mean [index.number_of_replicas]?");
+    }
+
     public void testValidate() {
         IndexScopedSettings settings = new IndexScopedSettings(
             Settings.EMPTY,
@@ -216,6 +229,13 @@ public class ScopedSettingsTests extends ESTestCase {
         } catch (IllegalArgumentException e) {
             assertEquals("Failed to parse value [true] for setting [index.number_of_replicas]", e.getMessage());
         }
+
+        try {
+            settings.validate("index.similarity.classic.type", Settings.builder().put("index.similarity.classic.type", "mine").build());
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("illegal value for [index.similarity.classic] cannot redefine built-in similarity", e.getMessage());
+        }
     }
 
 
@@ -240,22 +260,41 @@ public class ScopedSettingsTests extends ESTestCase {
 
         try {
             new IndexScopedSettings(
-                Settings.EMPTY, Collections.singleton(Setting.groupSetting("boo .", false, Setting.Scope.INDEX)));
+                Settings.EMPTY, Collections.singleton(Setting.groupSetting("foo.bar.", Property.IndexScope)));
             fail();
         } catch (IllegalArgumentException e) {
-            assertEquals("illegal settings key: [boo .]", e.getMessage());
+            assertEquals("illegal settings key: [foo.bar.] must start with [index.]", e.getMessage());
         }
-        new IndexScopedSettings(
-            Settings.EMPTY, Collections.singleton(Setting.groupSetting("boo.", false, Setting.Scope.INDEX)));
+
         try {
             new IndexScopedSettings(
-                Settings.EMPTY, Collections.singleton(Setting.boolSetting("boo.", true, false, Setting.Scope.INDEX)));
+                Settings.EMPTY, Collections.singleton(Setting.simpleString("foo.bar", Property.IndexScope)));
             fail();
         } catch (IllegalArgumentException e) {
-            assertEquals("illegal settings key: [boo.]", e.getMessage());
+            assertEquals("illegal settings key: [foo.bar] must start with [index.]", e.getMessage());
+        }
+
+        try {
+            new IndexScopedSettings(
+                Settings.EMPTY, Collections.singleton(Setting.groupSetting("index. foo.", Property.IndexScope)));
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("illegal settings key: [index. foo.]", e.getMessage());
         }
         new IndexScopedSettings(
-            Settings.EMPTY, Collections.singleton(Setting.boolSetting("boo", true, false, Setting.Scope.INDEX)));
+            Settings.EMPTY, Collections.singleton(Setting.groupSetting("index.", Property.IndexScope)));
+        try {
+            new IndexScopedSettings(
+                Settings.EMPTY, Collections.singleton(Setting.boolSetting("index.", true, Property.IndexScope)));
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("illegal settings key: [index.]", e.getMessage());
+        }
+        new IndexScopedSettings(
+            Settings.EMPTY, Collections.singleton(Setting.boolSetting("index.boo", true, Property.IndexScope)));
+
+        new ClusterSettings(
+            Settings.EMPTY, Collections.singleton(Setting.boolSetting("index.boo", true, Property.NodeScope)));
     }
 
     public void testLoggingUpdates() {
@@ -306,9 +345,9 @@ public class ScopedSettingsTests extends ESTestCase {
     public void testOverlappingComplexMatchSettings() {
         Set<Setting<?>> settings = new LinkedHashSet<>(2);
         final boolean groupFirst = randomBoolean();
-        final Setting<?> groupSetting = Setting.groupSetting("foo.", false, Setting.Scope.CLUSTER);
-        final Setting<?> listSetting = Setting.listSetting("foo.bar", Collections.emptyList(), Function.identity(), false,
-            Setting.Scope.CLUSTER);
+        final Setting<?> groupSetting = Setting.groupSetting("foo.", Property.NodeScope);
+        final Setting<?> listSetting =
+            Setting.listSetting("foo.bar", Collections.emptyList(), Function.identity(), Property.NodeScope);
         settings.add(groupFirst ? groupSetting : listSetting);
         settings.add(groupFirst ? listSetting : groupSetting);
 
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
index df2014f7855..14fdcb1e0ac 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.common.settings;
 
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
@@ -28,24 +29,29 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
 
 public class SettingTests extends ESTestCase {
 
 
     public void testGet() {
-        Setting<Boolean> booleanSetting = Setting.boolSetting("foo.bar", false, true, Setting.Scope.CLUSTER);
+        Setting<Boolean> booleanSetting = Setting.boolSetting("foo.bar", false, Property.Dynamic, Property.NodeScope);
         assertFalse(booleanSetting.get(Settings.EMPTY));
         assertFalse(booleanSetting.get(Settings.builder().put("foo.bar", false).build()));
         assertTrue(booleanSetting.get(Settings.builder().put("foo.bar", true).build()));
     }
 
     public void testByteSize() {
-        Setting<ByteSizeValue> byteSizeValueSetting = Setting.byteSizeSetting("a.byte.size", new ByteSizeValue(1024), true, Setting.Scope.CLUSTER);
+        Setting<ByteSizeValue> byteSizeValueSetting =
+            Setting.byteSizeSetting("a.byte.size", new ByteSizeValue(1024), Property.Dynamic, Property.NodeScope);
         assertFalse(byteSizeValueSetting.isGroupSetting());
         ByteSizeValue byteSizeValue = byteSizeValueSetting.get(Settings.EMPTY);
         assertEquals(byteSizeValue.bytes(), 1024);
 
-        byteSizeValueSetting = Setting.byteSizeSetting("a.byte.size", s -> "2048b", true, Setting.Scope.CLUSTER);
+        byteSizeValueSetting = Setting.byteSizeSetting("a.byte.size", s -> "2048b", Property.Dynamic, Property.NodeScope);
         byteSizeValue = byteSizeValueSetting.get(Settings.EMPTY);
         assertEquals(byteSizeValue.bytes(), 2048);
 
@@ -64,7 +70,7 @@ public class SettingTests extends ESTestCase {
     }
 
     public void testSimpleUpdate() {
-        Setting<Boolean> booleanSetting = Setting.boolSetting("foo.bar", false, true, Setting.Scope.CLUSTER);
+        Setting<Boolean> booleanSetting = Setting.boolSetting("foo.bar", false, Property.Dynamic, Property.NodeScope);
         AtomicReference<Boolean> atomicBoolean = new AtomicReference<>(null);
         ClusterSettings.SettingUpdater<Boolean> settingUpdater = booleanSetting.newUpdater(atomicBoolean::set, logger);
         Settings build = Settings.builder().put("foo.bar", false).build();
@@ -85,7 +91,7 @@ public class SettingTests extends ESTestCase {
     }
 
     public void testUpdateNotDynamic() {
-        Setting<Boolean> booleanSetting = Setting.boolSetting("foo.bar", false, false, Setting.Scope.CLUSTER);
+        Setting<Boolean> booleanSetting = Setting.boolSetting("foo.bar", false, Property.NodeScope);
         assertFalse(booleanSetting.isGroupSetting());
         AtomicReference<Boolean> atomicBoolean = new AtomicReference<>(null);
         try {
@@ -97,7 +103,7 @@ public class SettingTests extends ESTestCase {
     }
 
     public void testUpdaterIsIsolated() {
-        Setting<Boolean> booleanSetting = Setting.boolSetting("foo.bar", false, true, Setting.Scope.CLUSTER);
+        Setting<Boolean> booleanSetting = Setting.boolSetting("foo.bar", false, Property.Dynamic, Property.NodeScope);
         AtomicReference<Boolean> ab1 = new AtomicReference<>(null);
         AtomicReference<Boolean> ab2 = new AtomicReference<>(null);
         ClusterSettings.SettingUpdater<Boolean> settingUpdater = booleanSetting.newUpdater(ab1::set, logger);
@@ -108,24 +114,28 @@ public class SettingTests extends ESTestCase {
 
     public void testDefault() {
         TimeValue defautlValue = TimeValue.timeValueMillis(randomIntBetween(0, 1000000));
-        Setting<TimeValue> setting = Setting.positiveTimeSetting("my.time.value", defautlValue, randomBoolean(), Setting.Scope.CLUSTER);
+        Setting<TimeValue> setting =
+            Setting.positiveTimeSetting("my.time.value", defautlValue, Property.NodeScope);
         assertFalse(setting.isGroupSetting());
         String aDefault = setting.getDefaultRaw(Settings.EMPTY);
         assertEquals(defautlValue.millis() + "ms", aDefault);
         assertEquals(defautlValue.millis(), setting.get(Settings.EMPTY).millis());
         assertEquals(defautlValue, setting.getDefault(Settings.EMPTY));
 
-        Setting<String> secondaryDefault = new Setting<>("foo.bar", (s) -> s.get("old.foo.bar", "some_default"), (s) -> s, randomBoolean(), Setting.Scope.CLUSTER);
+        Setting<String> secondaryDefault =
+            new Setting<>("foo.bar", (s) -> s.get("old.foo.bar", "some_default"), Function.identity(), Property.NodeScope);
         assertEquals("some_default", secondaryDefault.get(Settings.EMPTY));
         assertEquals("42", secondaryDefault.get(Settings.builder().put("old.foo.bar", 42).build()));
-        Setting<String> secondaryDefaultViaSettings = new Setting<>("foo.bar", secondaryDefault, (s) -> s, randomBoolean(), Setting.Scope.CLUSTER);
+        Setting<String> secondaryDefaultViaSettings =
+            new Setting<>("foo.bar", secondaryDefault, Function.identity(), Property.NodeScope);
         assertEquals("some_default", secondaryDefaultViaSettings.get(Settings.EMPTY));
         assertEquals("42", secondaryDefaultViaSettings.get(Settings.builder().put("old.foo.bar", 42).build()));
     }
 
     public void testComplexType() {
         AtomicReference<ComplexType> ref = new AtomicReference<>(null);
-        Setting<ComplexType> setting = new Setting<>("foo.bar", (s) -> "", (s) -> new ComplexType(s), true, Setting.Scope.CLUSTER);
+        Setting<ComplexType> setting = new Setting<>("foo.bar", (s) -> "", (s) -> new ComplexType(s),
+            Property.Dynamic, Property.NodeScope);
         assertFalse(setting.isGroupSetting());
         ref.set(setting.get(Settings.EMPTY));
         ComplexType type = ref.get();
@@ -146,15 +156,17 @@ public class SettingTests extends ESTestCase {
     }
 
     public void testType() {
-        Setting<Integer> integerSetting = Setting.intSetting("foo.int.bar", 1, true, Setting.Scope.CLUSTER);
-        assertEquals(integerSetting.getScope(), Setting.Scope.CLUSTER);
-        integerSetting = Setting.intSetting("foo.int.bar", 1, true, Setting.Scope.INDEX);
-        assertEquals(integerSetting.getScope(), Setting.Scope.INDEX);
+        Setting<Integer> integerSetting = Setting.intSetting("foo.int.bar", 1, Property.Dynamic, Property.NodeScope);
+        assertThat(integerSetting.hasNodeScope(), is(true));
+        assertThat(integerSetting.hasIndexScope(), is(false));
+        integerSetting = Setting.intSetting("foo.int.bar", 1, Property.Dynamic, Property.IndexScope);
+        assertThat(integerSetting.hasIndexScope(), is(true));
+        assertThat(integerSetting.hasNodeScope(), is(false));
     }
 
     public void testGroups() {
         AtomicReference<Settings> ref = new AtomicReference<>(null);
-        Setting<Settings> setting = Setting.groupSetting("foo.bar.", true, Setting.Scope.CLUSTER);
+        Setting<Settings> setting = Setting.groupSetting("foo.bar.", Property.Dynamic, Property.NodeScope);
         assertTrue(setting.isGroupSetting());
         ClusterSettings.SettingUpdater<Settings> settingUpdater = setting.newUpdater(ref::set, logger);
 
@@ -232,8 +244,8 @@ public class SettingTests extends ESTestCase {
 
     public void testComposite() {
         Composite c = new Composite();
-        Setting<Integer> a = Setting.intSetting("foo.int.bar.a", 1, true, Setting.Scope.CLUSTER);
-        Setting<Integer> b = Setting.intSetting("foo.int.bar.b", 1, true, Setting.Scope.CLUSTER);
+        Setting<Integer> a = Setting.intSetting("foo.int.bar.a", 1, Property.Dynamic, Property.NodeScope);
+        Setting<Integer> b = Setting.intSetting("foo.int.bar.b", 1, Property.Dynamic, Property.NodeScope);
         ClusterSettings.SettingUpdater<Tuple<Integer, Integer>> settingUpdater = Setting.compoundUpdater(c::set, a, b, logger);
         assertFalse(settingUpdater.apply(Settings.EMPTY, Settings.EMPTY));
         assertNull(c.a);
@@ -261,7 +273,8 @@ public class SettingTests extends ESTestCase {
     }
 
     public void testListSettings() {
-        Setting<List<String>> listSetting = Setting.listSetting("foo.bar", Arrays.asList("foo,bar"), (s) -> s.toString(), true, Setting.Scope.CLUSTER);
+        Setting<List<String>> listSetting = Setting.listSetting("foo.bar", Arrays.asList("foo,bar"), (s) -> s.toString(),
+            Property.Dynamic, Property.NodeScope);
         List<String> value = listSetting.get(Settings.EMPTY);
         assertEquals(1, value.size());
         assertEquals("foo,bar", value.get(0));
@@ -300,7 +313,8 @@ public class SettingTests extends ESTestCase {
         assertEquals(1, ref.get().size());
         assertEquals("foo,bar", ref.get().get(0));
 
-        Setting<List<Integer>> otherSettings = Setting.listSetting("foo.bar", Collections.emptyList(), Integer::parseInt, true, Setting.Scope.CLUSTER);
+        Setting<List<Integer>> otherSettings = Setting.listSetting("foo.bar", Collections.emptyList(), Integer::parseInt,
+            Property.Dynamic, Property.NodeScope);
         List<Integer> defaultValue = otherSettings.get(Settings.EMPTY);
         assertEquals(0, defaultValue.size());
         List<Integer> intValues = otherSettings.get(Settings.builder().put("foo.bar", "0,1,2,3").build());
@@ -309,7 +323,8 @@ public class SettingTests extends ESTestCase {
             assertEquals(i, intValues.get(i).intValue());
         }
 
-        Setting<List<String>> settingWithFallback = Setting.listSetting("foo.baz", listSetting, s -> s, true, Setting.Scope.CLUSTER);
+        Setting<List<String>> settingWithFallback = Setting.listSetting("foo.baz", listSetting, Function.identity(),
+            Property.Dynamic, Property.NodeScope);
         value = settingWithFallback.get(Settings.EMPTY);
         assertEquals(1, value.size());
         assertEquals("foo,bar", value.get(0));
@@ -331,7 +346,8 @@ public class SettingTests extends ESTestCase {
     }
 
     public void testListSettingAcceptsNumberSyntax() {
-        Setting<List<String>> listSetting = Setting.listSetting("foo.bar", Arrays.asList("foo,bar"), (s) -> s.toString(), true, Setting.Scope.CLUSTER);
+        Setting<List<String>> listSetting = Setting.listSetting("foo.bar", Arrays.asList("foo,bar"), (s) -> s.toString(),
+            Property.Dynamic, Property.NodeScope);
         List<String> input = Arrays.asList("test", "test1, test2", "test", ",,,,");
         Settings.Builder builder = Settings.builder().putArray("foo.bar", input.toArray(new String[0]));
         // try to parse this really annoying format
@@ -348,8 +364,8 @@ public class SettingTests extends ESTestCase {
         assertTrue(listSetting.match("foo.bar." + randomIntBetween(0,10000)));
     }
 
-    public void testPrefixKeySetting() {
-        Setting<Boolean> setting = Setting.prefixKeySetting("foo.", "false", Boolean::parseBoolean, false, Setting.Scope.CLUSTER);
+    public void testDynamicKeySetting() {
+        Setting<Boolean> setting = Setting.prefixKeySetting("foo.", "false", Boolean::parseBoolean, Property.NodeScope);
         assertTrue(setting.hasComplexMatcher());
         assertTrue(setting.match("foo.bar"));
         assertFalse(setting.match("foo"));
@@ -366,7 +382,8 @@ public class SettingTests extends ESTestCase {
     }
 
     public void testAdfixKeySetting() {
-        Setting<Boolean> setting = Setting.adfixKeySetting("foo", "enable", "false", Boolean::parseBoolean, false, Setting.Scope.CLUSTER);
+        Setting<Boolean> setting =
+            Setting.adfixKeySetting("foo", "enable", "false", Boolean::parseBoolean, Property.NodeScope);
         assertTrue(setting.hasComplexMatcher());
         assertTrue(setting.match("foo.bar.enable"));
         assertTrue(setting.match("foo.baz.enable"));
@@ -387,7 +404,7 @@ public class SettingTests extends ESTestCase {
     }
 
     public void testMinMaxInt() {
-        Setting<Integer> integerSetting = Setting.intSetting("foo.bar", 1, 0, 10, false, Setting.Scope.CLUSTER);
+        Setting<Integer> integerSetting = Setting.intSetting("foo.bar", 1, 0, 10, Property.NodeScope);
         try {
             integerSetting.get(Settings.builder().put("foo.bar", 11).build());
             fail();
@@ -405,4 +422,39 @@ public class SettingTests extends ESTestCase {
         assertEquals(5, integerSetting.get(Settings.builder().put("foo.bar", 5).build()).intValue());
         assertEquals(1, integerSetting.get(Settings.EMPTY).intValue());
     }
+
+    /**
+     * Only one single scope can be added to any setting
+     */
+    public void testMutuallyExclusiveScopes() {
+        // Those should pass
+        Setting<String> setting = Setting.simpleString("foo.bar", Property.NodeScope);
+        assertThat(setting.hasNodeScope(), is(true));
+        assertThat(setting.hasIndexScope(), is(false));
+        setting = Setting.simpleString("foo.bar", Property.IndexScope);
+        assertThat(setting.hasIndexScope(), is(true));
+        assertThat(setting.hasNodeScope(), is(false));
+
+        // We accept settings with no scope but they will be rejected when we register with SettingsModule.registerSetting
+        setting = Setting.simpleString("foo.bar");
+        assertThat(setting.hasIndexScope(), is(false));
+        assertThat(setting.hasNodeScope(), is(false));
+
+        // We accept settings with multiple scopes but they will be rejected when we register with SettingsModule.registerSetting
+        setting = Setting.simpleString("foo.bar", Property.IndexScope, Property.NodeScope);
+        assertThat(setting.hasIndexScope(), is(true));
+        assertThat(setting.hasNodeScope(), is(true));
+    }
+
+    /**
+     * We can't have Null properties
+     */
+    public void testRejectNullProperties() {
+        try {
+            Setting.simpleString("foo.bar", (Property[]) null);
+            fail();
+        } catch (IllegalArgumentException ex) {
+            assertThat(ex.getMessage(), containsString("properties can not be null for setting"));
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
index 4f790c2d3a9..7401164a4f6 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
@@ -20,6 +20,10 @@
 package org.elasticsearch.common.settings;
 
 import org.elasticsearch.common.inject.ModuleTestCase;
+import org.elasticsearch.common.settings.Setting.Property;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
 
 public class SettingsModuleTests extends ModuleTestCase {
 
@@ -32,12 +36,35 @@ public class SettingsModuleTests extends ModuleTestCase {
         {
             Settings settings = Settings.builder().put("cluster.routing.allocation.balance.shard", "[2.0]").build();
             SettingsModule module = new SettingsModule(settings);
-            try {
-                assertInstanceBinding(module, Settings.class, (s) -> s == settings);
-                fail();
-            } catch (IllegalArgumentException ex) {
-                assertEquals("Failed to parse value [[2.0]] for setting [cluster.routing.allocation.balance.shard]", ex.getMessage());
-            }
+            IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+                () -> assertInstanceBinding(module, Settings.class, (s) -> s == settings));
+            assertEquals("Failed to parse value [[2.0]] for setting [cluster.routing.allocation.balance.shard]", ex.getMessage());
+        }
+
+        {
+            Settings settings = Settings.builder().put("cluster.routing.allocation.balance.shard", "[2.0]")
+                .put("some.foo.bar", 1).build();
+            SettingsModule module = new SettingsModule(settings);
+            IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+                () -> assertInstanceBinding(module, Settings.class, (s) -> s == settings));
+            assertEquals("Failed to parse value [[2.0]] for setting [cluster.routing.allocation.balance.shard]", ex.getMessage());
+            assertEquals(1, ex.getSuppressed().length);
+            assertEquals("unknown setting [some.foo.bar]", ex.getSuppressed()[0].getMessage());
+        }
+
+        {
+            Settings settings = Settings.builder().put("index.codec", "default")
+                .put("index.foo.bar", 1).build();
+            SettingsModule module = new SettingsModule(settings);
+            IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+                () -> assertInstanceBinding(module, Settings.class, (s) -> s == settings));
+            assertEquals("node settings must not contain any index level settings", ex.getMessage());
+        }
+
+        {
+            Settings settings = Settings.builder().put("index.codec", "default").build();
+            SettingsModule module = new SettingsModule(settings);
+            assertInstanceBinding(module, Settings.class, (s) -> s == settings);
         }
     }
 
@@ -45,13 +72,13 @@ public class SettingsModuleTests extends ModuleTestCase {
         {
             Settings settings = Settings.builder().put("some.custom.setting", "2.0").build();
             SettingsModule module = new SettingsModule(settings);
-            module.registerSetting(Setting.floatSetting("some.custom.setting", 1.0f, false, Setting.Scope.CLUSTER));
+            module.registerSetting(Setting.floatSetting("some.custom.setting", 1.0f, Property.NodeScope));
             assertInstanceBinding(module, Settings.class, (s) -> s == settings);
         }
         {
             Settings settings = Settings.builder().put("some.custom.setting", "false").build();
             SettingsModule module = new SettingsModule(settings);
-            module.registerSetting(Setting.floatSetting("some.custom.setting", 1.0f, false, Setting.Scope.CLUSTER));
+            module.registerSetting(Setting.floatSetting("some.custom.setting", 1.0f, Property.NodeScope));
             try {
                 assertInstanceBinding(module, Settings.class, (s) -> s == settings);
                 fail();
@@ -131,12 +158,11 @@ public class SettingsModuleTests extends ModuleTestCase {
     public void testRegisterSettingsFilter() {
         Settings settings = Settings.builder().put("foo.bar", "false").put("bar.foo", false).put("bar.baz", false).build();
         SettingsModule module = new SettingsModule(settings);
-        module.registerSetting(Setting.boolSetting("foo.bar", true, false, Setting.Scope.CLUSTER));
-        module.registerSetting(Setting.boolSetting("bar.foo", true, false, Setting.Scope.CLUSTER));
-        module.registerSetting(Setting.boolSetting("bar.baz", true, false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.boolSetting("foo.bar", true, Property.NodeScope));
+        module.registerSetting(Setting.boolSetting("bar.foo", true, Property.NodeScope, Property.Filtered));
+        module.registerSetting(Setting.boolSetting("bar.baz", true, Property.NodeScope));
 
         module.registerSettingsFilter("foo.*");
-        module.registerSettingsFilterIfMissing("bar.foo");
         try {
             module.registerSettingsFilter("bar.foo");
             fail();
@@ -149,4 +175,34 @@ public class SettingsModuleTests extends ModuleTestCase {
         assertInstanceBinding(module, SettingsFilter.class, (s) -> s.filter(settings).getAsMap().get("bar.baz").equals("false"));
 
     }
+
+    public void testMutuallyExclusiveScopes() {
+        new SettingsModule(Settings.EMPTY).registerSetting(Setting.simpleString("foo.bar", Property.NodeScope));
+        new SettingsModule(Settings.EMPTY).registerSetting(Setting.simpleString("foo.bar", Property.IndexScope));
+
+        // Those should fail
+        try {
+            new SettingsModule(Settings.EMPTY).registerSetting(Setting.simpleString("foo.bar"));
+            fail("No scope should fail");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), containsString("No scope found for setting"));
+        }
+        // Some settings have both scopes - that's fine too if they have per-node defaults
+        SettingsModule module = new SettingsModule(Settings.EMPTY);
+        module.registerSetting(Setting.simpleString("foo.bar", Property.IndexScope, Property.NodeScope));
+
+        try {
+            module.registerSetting(Setting.simpleString("foo.bar", Property.NodeScope));
+            fail("already registered");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), containsString("Cannot register setting [foo.bar] twice"));
+        }
+
+        try {
+            module.registerSetting(Setting.simpleString("foo.bar", Property.IndexScope));
+            fail("already registered");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), containsString("Cannot register setting [foo.bar] twice"));
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
index e3f2bc1bb2b..eb6cc56816f 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
@@ -201,8 +201,8 @@ public class SettingsTests extends ESTestCase {
         assertThat(settings.getAsArray("value"), arrayContaining("2", "3"));
 
         settings = settingsBuilder()
-                .put(new YamlSettingsLoader().load("value: 1"))
-                .put(new YamlSettingsLoader().load("value: [ 2, 3 ]"))
+                .put(new YamlSettingsLoader(false).load("value: 1"))
+                .put(new YamlSettingsLoader(false).load("value: [ 2, 3 ]"))
                 .build();
         assertThat(settings.getAsArray("value"), arrayContaining("2", "3"));
 
diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java
index d7f10891f28..154ef8ee039 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java
@@ -25,15 +25,14 @@ import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
+import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
-/**
- *
- */
 public class JsonSettingsLoaderTests extends ESTestCase {
+
     public void testSimpleJsonSettings() throws Exception {
-        String json = "/org/elasticsearch/common/settings/loader/test-settings.json";
-        Settings settings = settingsBuilder()
+        final String json = "/org/elasticsearch/common/settings/loader/test-settings.json";
+        final Settings settings = settingsBuilder()
                 .loadFromStream(json, getClass().getResourceAsStream(json))
                 .build();
 
@@ -50,15 +49,23 @@ public class JsonSettingsLoaderTests extends ESTestCase {
     }
 
     public void testDuplicateKeysThrowsException() {
-        String json = "{\"foo\":\"bar\",\"foo\":\"baz\"}";
-        try {
-            settingsBuilder()
-                    .loadFromSource(json)
-                    .build();
-            fail("expected exception");
-        } catch (SettingsException e) {
-            assertEquals(e.getCause().getClass(), ElasticsearchParseException.class);
-            assertTrue(e.toString().contains("duplicate settings key [foo] found at line number [1], column number [20], previous value [bar], current value [baz]"));
-        }
+        final String json = "{\"foo\":\"bar\",\"foo\":\"baz\"}";
+        final SettingsException e = expectThrows(SettingsException.class, () -> settingsBuilder().loadFromSource(json).build());
+        assertEquals(e.getCause().getClass(), ElasticsearchParseException.class);
+        assertThat(
+                e.toString(),
+                containsString("duplicate settings key [foo] " +
+                        "found at line number [1], " +
+                        "column number [20], " +
+                        "previous value [bar], " +
+                        "current value [baz]"));
     }
+
+    public void testNullValuedSettingThrowsException() {
+        final String json = "{\"foo\":null}";
+        final ElasticsearchParseException e =
+                expectThrows(ElasticsearchParseException.class, () -> new JsonSettingsLoader(false).load(json));
+        assertThat(e.toString(), containsString("null-valued setting found for key [foo] found at line number [1], column number [8]"));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/PropertiesSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/PropertiesSettingsLoaderTests.java
index 7a1897fbaf9..c13ae7cc68b 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/loader/PropertiesSettingsLoaderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/loader/PropertiesSettingsLoaderTests.java
@@ -21,27 +21,37 @@ package org.elasticsearch.common.settings.loader;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.nio.charset.Charset;
 
 public class PropertiesSettingsLoaderTests extends ESTestCase {
+
+    private PropertiesSettingsLoader loader;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        loader = new PropertiesSettingsLoader();
+    }
+
     public void testDuplicateKeyFromStringThrowsException() throws IOException {
-        PropertiesSettingsLoader loader = new PropertiesSettingsLoader();
-        try {
-            loader.load("foo=bar\nfoo=baz");
-            fail("expected exception");
-        } catch (ElasticsearchParseException e) {
-            assertEquals(e.getMessage(), "duplicate settings key [foo] found, previous value [bar], current value [baz]");
-        }
+        final ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> loader.load("foo=bar\nfoo=baz"));
+        assertEquals(e.getMessage(), "duplicate settings key [foo] found, previous value [bar], current value [baz]");
     }
 
     public void testDuplicateKeysFromBytesThrowsException() throws IOException {
-        PropertiesSettingsLoader loader = new PropertiesSettingsLoader();
-        try {
-            loader.load("foo=bar\nfoo=baz".getBytes(Charset.defaultCharset()));
-        } catch (ElasticsearchParseException e) {
-            assertEquals(e.getMessage(), "duplicate settings key [foo] found, previous value [bar], current value [baz]");
-        }
+        final ElasticsearchParseException e = expectThrows(
+                ElasticsearchParseException.class,
+                () -> loader.load("foo=bar\nfoo=baz".getBytes(Charset.defaultCharset()))
+        );
+        assertEquals(e.getMessage(), "duplicate settings key [foo] found, previous value [bar], current value [baz]");
     }
+
+    public void testThatNoDuplicatesPropertiesDoesNotAcceptNullValues() {
+        final PropertiesSettingsLoader.NoDuplicatesProperties properties = loader.new NoDuplicatesProperties();
+        expectThrows(NullPointerException.class, () -> properties.put("key", null));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
index 48703044ecd..2e2a187da0b 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
@@ -28,13 +28,11 @@ import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
-/**
- *
- */
 public class YamlSettingsLoaderTests extends ESTestCase {
+
     public void testSimpleYamlSettings() throws Exception {
-        String yaml = "/org/elasticsearch/common/settings/loader/test-settings.yml";
-        Settings settings = settingsBuilder()
+        final String yaml = "/org/elasticsearch/common/settings/loader/test-settings.yml";
+        final Settings settings = settingsBuilder()
                 .loadFromStream(yaml, getClass().getResourceAsStream(yaml))
                 .build();
 
@@ -51,39 +49,41 @@ public class YamlSettingsLoaderTests extends ESTestCase {
     }
 
     public void testIndentation() {
-        String yaml = "/org/elasticsearch/common/settings/loader/indentation-settings.yml";
-        try {
-            settingsBuilder()
-                .loadFromStream(yaml, getClass().getResourceAsStream(yaml))
-                .build();
-            fail("Expected SettingsException");
-        } catch(SettingsException e ) {
-            assertThat(e.getMessage(), containsString("Failed to load settings"));
-        }
+        final String yaml = "/org/elasticsearch/common/settings/loader/indentation-settings.yml";
+        final SettingsException e =
+                expectThrows(
+                        SettingsException.class,
+                        () -> settingsBuilder().loadFromStream(yaml, getClass().getResourceAsStream(yaml)).build());
+        assertThat(e.getMessage(), containsString("Failed to load settings"));
     }
 
     public void testIndentationWithExplicitDocumentStart() {
-        String yaml = "/org/elasticsearch/common/settings/loader/indentation-with-explicit-document-start-settings.yml";
-        try {
-            settingsBuilder()
-                    .loadFromStream(yaml, getClass().getResourceAsStream(yaml))
-                    .build();
-            fail("Expected SettingsException");
-        } catch (SettingsException e) {
-            assertThat(e.getMessage(), containsString("Failed to load settings"));
-        }
+        final String yaml = "/org/elasticsearch/common/settings/loader/indentation-with-explicit-document-start-settings.yml";
+        final SettingsException e =
+                expectThrows(
+                        SettingsException.class,
+                        () -> settingsBuilder().loadFromStream(yaml, getClass().getResourceAsStream(yaml)).build());
+        assertThat(e.getMessage(), containsString("Failed to load settings"));
     }
 
     public void testDuplicateKeysThrowsException() {
-        String yaml = "foo: bar\nfoo: baz";
-        try {
-            settingsBuilder()
-                    .loadFromSource(yaml)
-                    .build();
-            fail("expected exception");
-        } catch (SettingsException e) {
-            assertEquals(e.getCause().getClass(), ElasticsearchParseException.class);
-            assertTrue(e.toString().contains("duplicate settings key [foo] found at line number [2], column number [6], previous value [bar], current value [baz]"));
-        }
+        final String yaml = "foo: bar\nfoo: baz";
+        final SettingsException e = expectThrows(SettingsException.class, () -> settingsBuilder().loadFromSource(yaml).build());
+        assertEquals(e.getCause().getClass(), ElasticsearchParseException.class);
+        assertThat(
+                e.toString(),
+                containsString("duplicate settings key [foo] " +
+                        "found at line number [2], " +
+                        "column number [6], " +
+                        "previous value [bar], " +
+                        "current value [baz]"));
     }
+
+    public void testNullValuedSettingThrowsException() {
+        final String yaml = "foo:";
+        final ElasticsearchParseException e =
+                expectThrows(ElasticsearchParseException.class, () -> new YamlSettingsLoader(false).load(yaml));
+        assertThat(e.toString(), containsString("null-valued setting found for key [foo] found at line number [1], column number [5]"));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java b/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java
index 5d7bbb3ca18..f9a4d3f22af 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java
@@ -83,7 +83,7 @@ public class DistanceUnitTests extends ESTestCase {
           try (BytesStreamOutput out = new BytesStreamOutput()) {
               unit.writeTo(out);
               try (StreamInput in = StreamInput.wrap(out.bytes())) {
-                  assertThat("Roundtrip serialisation failed.", DistanceUnit.readDistanceUnit(in), equalTo(unit));
+                  assertThat("Roundtrip serialisation failed.", DistanceUnit.readFromStream(in), equalTo(unit));
               }
           }
         }
diff --git a/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java b/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java
new file mode 100644
index 00000000000..b158b961d9a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java
@@ -0,0 +1,345 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.util;
+
+import org.apache.lucene.util.CollectionUtil;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
+import org.elasticsearch.Version;
+import org.elasticsearch.bwcompat.OldIndexBackwardsCompatibilityIT;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.AllocationId;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.gateway.MetaDataStateFormat;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.shard.ShardPath;
+import org.elasticsearch.index.shard.ShardStateMetaData;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.BufferedWriter;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+@LuceneTestCase.SuppressFileSystems("ExtrasFS")
+public class IndexFolderUpgraderTests extends ESTestCase {
+
+    /**
+     * tests custom data paths are upgraded
+     */
+    public void testUpgradeCustomDataPath() throws IOException {
+        Path customPath = createTempDir();
+        final Settings nodeSettings = Settings.builder()
+            .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), randomBoolean())
+            .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), customPath.toAbsolutePath().toString()).build();
+        try (NodeEnvironment nodeEnv = newNodeEnvironment(nodeSettings)) {
+            final Index index = new Index(randomAsciiOfLength(10), Strings.randomBase64UUID());
+            Settings settings = Settings.builder()
+                .put(nodeSettings)
+                .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID())
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_0_0)
+                .put(IndexMetaData.SETTING_DATA_PATH, customPath.toAbsolutePath().toString())
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 5))
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .build();
+            IndexMetaData indexState = IndexMetaData.builder(index.getName()).settings(settings).build();
+            int numIdxFiles = randomIntBetween(1, 5);
+            int numTranslogFiles = randomIntBetween(1, 5);
+            IndexSettings indexSettings = new IndexSettings(indexState, nodeSettings);
+            writeIndex(nodeEnv, indexSettings, numIdxFiles, numTranslogFiles);
+            IndexFolderUpgrader helper = new IndexFolderUpgrader(settings, nodeEnv);
+            helper.upgrade(indexSettings.getIndex().getName());
+            checkIndex(nodeEnv, indexSettings, numIdxFiles, numTranslogFiles);
+        }
+    }
+
+    /**
+     * tests upgrade on partially upgraded index, when we crash while upgrading
+     */
+    public void testPartialUpgradeCustomDataPath() throws IOException {
+        Path customPath = createTempDir();
+        final Settings nodeSettings = Settings.builder()
+            .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), randomBoolean())
+            .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), customPath.toAbsolutePath().toString()).build();
+        try (NodeEnvironment nodeEnv = newNodeEnvironment(nodeSettings)) {
+            final Index index = new Index(randomAsciiOfLength(10), Strings.randomBase64UUID());
+            Settings settings = Settings.builder()
+                .put(nodeSettings)
+                .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID())
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_0_0)
+                .put(IndexMetaData.SETTING_DATA_PATH, customPath.toAbsolutePath().toString())
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 5))
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .build();
+            IndexMetaData indexState = IndexMetaData.builder(index.getName()).settings(settings).build();
+            int numIdxFiles = randomIntBetween(1, 5);
+            int numTranslogFiles = randomIntBetween(1, 5);
+            IndexSettings indexSettings = new IndexSettings(indexState, nodeSettings);
+            writeIndex(nodeEnv, indexSettings, numIdxFiles, numTranslogFiles);
+            IndexFolderUpgrader helper = new IndexFolderUpgrader(settings, nodeEnv) {
+                @Override
+                void upgrade(Index index, Path source, Path target) throws IOException {
+                    if(randomBoolean()) {
+                        throw new FileNotFoundException("simulated");
+                    }
+                }
+            };
+            // only upgrade some paths
+            try {
+                helper.upgrade(index.getName());
+            } catch (IOException e) {
+                assertTrue(e instanceof FileNotFoundException);
+            }
+            helper = new IndexFolderUpgrader(settings, nodeEnv);
+            // try to upgrade again
+            helper.upgrade(indexSettings.getIndex().getName());
+            checkIndex(nodeEnv, indexSettings, numIdxFiles, numTranslogFiles);
+        }
+    }
+
+    public void testUpgrade() throws IOException {
+        final Settings nodeSettings = Settings.builder()
+            .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), randomBoolean()).build();
+        try (NodeEnvironment nodeEnv = newNodeEnvironment(nodeSettings)) {
+            final Index index = new Index(randomAsciiOfLength(10), Strings.randomBase64UUID());
+            Settings settings = Settings.builder()
+                .put(nodeSettings)
+                .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID())
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_0_0)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 5))
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .build();
+            IndexMetaData indexState = IndexMetaData.builder(index.getName()).settings(settings).build();
+            int numIdxFiles = randomIntBetween(1, 5);
+            int numTranslogFiles = randomIntBetween(1, 5);
+            IndexSettings indexSettings = new IndexSettings(indexState, nodeSettings);
+            writeIndex(nodeEnv, indexSettings, numIdxFiles, numTranslogFiles);
+            IndexFolderUpgrader helper = new IndexFolderUpgrader(settings, nodeEnv);
+            helper.upgrade(indexSettings.getIndex().getName());
+            checkIndex(nodeEnv, indexSettings, numIdxFiles, numTranslogFiles);
+        }
+    }
+
+    public void testUpgradeIndices() throws IOException {
+        final Settings nodeSettings = Settings.builder()
+            .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), randomBoolean()).build();
+        try (NodeEnvironment nodeEnv = newNodeEnvironment(nodeSettings)) {
+            Map<IndexSettings, Tuple<Integer, Integer>>  indexSettingsMap = new HashMap<>();
+            for (int i = 0; i < randomIntBetween(2, 5); i++) {
+                final Index index = new Index(randomAsciiOfLength(10), Strings.randomBase64UUID());
+                Settings settings = Settings.builder()
+                    .put(nodeSettings)
+                    .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID())
+                    .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_0_0)
+                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 5))
+                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                    .build();
+                IndexMetaData indexState = IndexMetaData.builder(index.getName()).settings(settings).build();
+                Tuple<Integer, Integer> fileCounts = new Tuple<>(randomIntBetween(1, 5), randomIntBetween(1, 5));
+                IndexSettings indexSettings = new IndexSettings(indexState, nodeSettings);
+                indexSettingsMap.put(indexSettings, fileCounts);
+                writeIndex(nodeEnv, indexSettings, fileCounts.v1(), fileCounts.v2());
+            }
+            IndexFolderUpgrader.upgradeIndicesIfNeeded(nodeSettings, nodeEnv);
+            for (Map.Entry<IndexSettings, Tuple<Integer, Integer>> entry : indexSettingsMap.entrySet()) {
+                checkIndex(nodeEnv, entry.getKey(), entry.getValue().v1(), entry.getValue().v2());
+            }
+        }
+    }
+
+    /**
+     * Run upgrade on a real bwc index
+     */
+    public void testUpgradeRealIndex() throws IOException, URISyntaxException {
+        List<Path> indexes = new ArrayList<>();
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(getBwcIndicesPath(), "index-*.zip")) {
+            for (Path path : stream) {
+                indexes.add(path);
+            }
+        }
+        CollectionUtil.introSort(indexes, (o1, o2) -> o1.getFileName().compareTo(o2.getFileName()));
+        final Path path = randomFrom(indexes);
+        final String indexName = path.getFileName().toString().replace(".zip", "").toLowerCase(Locale.ROOT);
+        try (NodeEnvironment nodeEnvironment = newNodeEnvironment()) {
+            Path unzipDir = createTempDir();
+            Path unzipDataDir = unzipDir.resolve("data");
+            // decompress the index
+            try (InputStream stream = Files.newInputStream(path)) {
+                TestUtil.unzip(stream, unzipDir);
+            }
+            // check it is unique
+            assertTrue(Files.exists(unzipDataDir));
+            Path[] list = FileSystemUtils.files(unzipDataDir);
+            if (list.length != 1) {
+                throw new IllegalStateException("Backwards index must contain exactly one cluster but was " + list.length);
+            }
+            // the bwc scripts packs the indices under this path
+            Path src = list[0].resolve("nodes/0/indices/" + indexName);
+            assertTrue("[" + path + "] missing index dir: " + src.toString(), Files.exists(src));
+            final Path indicesPath = randomFrom(nodeEnvironment.nodePaths()).indicesPath;
+            logger.info("--> injecting index [{}] into [{}]", indexName, indicesPath);
+            OldIndexBackwardsCompatibilityIT.copyIndex(logger, src, indexName, indicesPath);
+            IndexFolderUpgrader.upgradeIndicesIfNeeded(Settings.EMPTY, nodeEnvironment);
+
+            // ensure old index folder is deleted
+            Set<String> indexFolders = nodeEnvironment.availableIndexFolders();
+            assertEquals(indexFolders.size(), 1);
+
+            // ensure index metadata is moved
+            IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger,
+                nodeEnvironment.resolveIndexFolder(indexFolders.iterator().next()));
+            assertNotNull(indexMetaData);
+            Index index = indexMetaData.getIndex();
+            assertEquals(index.getName(), indexName);
+
+            Set<ShardId> shardIds = nodeEnvironment.findAllShardIds(index);
+            // ensure all shards are moved
+            assertEquals(shardIds.size(), indexMetaData.getNumberOfShards());
+            for (ShardId shardId : shardIds) {
+                final ShardPath shardPath = ShardPath.loadShardPath(logger, nodeEnvironment, shardId,
+                    new IndexSettings(indexMetaData, Settings.EMPTY));
+                final Path translog = shardPath.resolveTranslog();
+                final Path idx = shardPath.resolveIndex();
+                final Path state = shardPath.getShardStatePath().resolve(MetaDataStateFormat.STATE_DIR_NAME);
+                assertTrue(shardPath.exists());
+                assertTrue(Files.exists(translog));
+                assertTrue(Files.exists(idx));
+                assertTrue(Files.exists(state));
+            }
+        }
+    }
+
+    public void testNeedsUpgrade() throws IOException {
+        final Index index = new Index("foo", Strings.randomBase64UUID());
+        IndexMetaData indexState = IndexMetaData.builder(index.getName())
+            .settings(Settings.builder()
+                .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID())
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .build();
+        try (NodeEnvironment nodeEnvironment = newNodeEnvironment()) {
+            IndexMetaData.FORMAT.write(indexState, 1, nodeEnvironment.indexPaths(index));
+            assertFalse(IndexFolderUpgrader.needsUpgrade(index, index.getUUID()));
+        }
+    }
+
+    private void checkIndex(NodeEnvironment nodeEnv, IndexSettings indexSettings,
+                            int numIdxFiles, int numTranslogFiles) throws IOException {
+        final Index index = indexSettings.getIndex();
+        // ensure index state can be loaded
+        IndexMetaData loadLatestState = IndexMetaData.FORMAT.loadLatestState(logger, nodeEnv.indexPaths(index));
+        assertNotNull(loadLatestState);
+        assertEquals(loadLatestState.getIndex(), index);
+        for (int shardId = 0; shardId < indexSettings.getNumberOfShards(); shardId++) {
+            // ensure shard path can be loaded
+            ShardPath targetShardPath = ShardPath.loadShardPath(logger, nodeEnv, new ShardId(index, shardId), indexSettings);
+            assertNotNull(targetShardPath);
+            // ensure shard contents are copied over
+            final Path translog = targetShardPath.resolveTranslog();
+            final Path idx = targetShardPath.resolveIndex();
+
+            // ensure index and translog files are copied over
+            assertEquals(numTranslogFiles, FileSystemUtils.files(translog).length);
+            assertEquals(numIdxFiles, FileSystemUtils.files(idx).length);
+            Path[] files = FileSystemUtils.files(translog);
+            final HashSet<Path> translogFiles = new HashSet<>(Arrays.asList(files));
+            for (int i = 0; i < numTranslogFiles; i++) {
+                final String name = Integer.toString(i);
+                translogFiles.contains(translog.resolve(name + ".translog"));
+                byte[] content = Files.readAllBytes(translog.resolve(name + ".translog"));
+                assertEquals(name , new String(content, StandardCharsets.UTF_8));
+            }
+            Path[] indexFileList = FileSystemUtils.files(idx);
+            final HashSet<Path> idxFiles = new HashSet<>(Arrays.asList(indexFileList));
+            for (int i = 0; i < numIdxFiles; i++) {
+                final String name = Integer.toString(i);
+                idxFiles.contains(idx.resolve(name + ".tst"));
+                byte[] content = Files.readAllBytes(idx.resolve(name + ".tst"));
+                assertEquals(name, new String(content, StandardCharsets.UTF_8));
+            }
+        }
+    }
+
+    private void writeIndex(NodeEnvironment nodeEnv, IndexSettings indexSettings,
+                            int numIdxFiles, int numTranslogFiles) throws IOException {
+        NodeEnvironment.NodePath[] nodePaths = nodeEnv.nodePaths();
+        Path[] oldIndexPaths = new Path[nodePaths.length];
+        for (int i = 0; i < nodePaths.length; i++) {
+            oldIndexPaths[i] = nodePaths[i].indicesPath.resolve(indexSettings.getIndex().getName());
+        }
+        IndexMetaData.FORMAT.write(indexSettings.getIndexMetaData(), 1, oldIndexPaths);
+        for (int id = 0; id < indexSettings.getNumberOfShards(); id++) {
+            Path oldIndexPath = randomFrom(oldIndexPaths);
+            ShardId shardId = new ShardId(indexSettings.getIndex(), id);
+            if (indexSettings.hasCustomDataPath()) {
+                Path customIndexPath = nodeEnv.resolveBaseCustomLocation(indexSettings).resolve(indexSettings.getIndex().getName());
+                writeShard(shardId, customIndexPath, numIdxFiles, numTranslogFiles);
+            } else {
+                writeShard(shardId, oldIndexPath, numIdxFiles, numTranslogFiles);
+            }
+            ShardStateMetaData state = new ShardStateMetaData(true, indexSettings.getUUID(), AllocationId.newInitializing());
+            ShardStateMetaData.FORMAT.write(state, 1, oldIndexPath.resolve(String.valueOf(shardId.getId())));
+        }
+    }
+
+    private void writeShard(ShardId shardId, Path indexLocation,
+                            final int numIdxFiles, final int numTranslogFiles) throws IOException {
+        Path oldShardDataPath = indexLocation.resolve(String.valueOf(shardId.getId()));
+        final Path translogPath = oldShardDataPath.resolve(ShardPath.TRANSLOG_FOLDER_NAME);
+        final Path idxPath = oldShardDataPath.resolve(ShardPath.INDEX_FOLDER_NAME);
+        Files.createDirectories(translogPath);
+        Files.createDirectories(idxPath);
+        for (int i = 0; i < numIdxFiles; i++) {
+            String filename = Integer.toString(i);
+            try (BufferedWriter w = Files.newBufferedWriter(idxPath.resolve(filename + ".tst"),
+                StandardCharsets.UTF_8)) {
+                w.write(filename);
+            }
+        }
+        for (int i = 0; i < numTranslogFiles; i++) {
+            String filename = Integer.toString(i);
+            try (BufferedWriter w = Files.newBufferedWriter(translogPath.resolve(filename + ".translog"),
+                StandardCharsets.UTF_8)) {
+                w.write(filename);
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
index cbf58bf9daa..1a582d48f6b 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
@@ -27,6 +27,8 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 
+import static org.hamcrest.Matchers.sameInstance;
+
 public class ThreadContextTests extends ESTestCase {
 
     public void testStashContext() {
@@ -235,4 +237,71 @@ public class ThreadContextTests extends ESTestCase {
         }
     }
 
+    public void testPreserveContext() throws IOException {
+        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+            Runnable withContext;
+
+            // Create a runnable that should run with some header
+            try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
+                threadContext.putHeader("foo", "bar");
+                withContext = threadContext.preserveContext(sometimesAbstractRunnable(() -> {
+                    assertEquals("bar", threadContext.getHeader("foo"));
+                }));
+            }
+
+            // We don't see the header outside of the runnable
+            assertNull(threadContext.getHeader("foo"));
+
+            // But we do inside of it
+            withContext.run();
+        }
+    }
+
+    public void testPreserveContextKeepsOriginalContextWhenCalledTwice() throws IOException {
+        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+            Runnable originalWithContext;
+            Runnable withContext;
+
+            // Create a runnable that should run with some header
+            try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
+                threadContext.putHeader("foo", "bar");
+                withContext = threadContext.preserveContext(sometimesAbstractRunnable(() -> {
+                    assertEquals("bar", threadContext.getHeader("foo"));
+                }));
+            }
+
+            // Now attempt to rewrap it
+            originalWithContext = withContext;
+            try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
+                threadContext.putHeader("foo", "zot");
+                withContext = threadContext.preserveContext(withContext);
+            }
+
+            // We get the original context inside the runnable
+            withContext.run();
+
+            // In fact the second wrapping didn't even change it
+            assertThat(withContext, sameInstance(originalWithContext));
+        }
+    }
+
+    /**
+     * Sometimes wraps a Runnable in an AbstractRunnable.
+     */
+    private Runnable sometimesAbstractRunnable(Runnable r) {
+        if (random().nextBoolean()) {
+            return r;
+        }
+        return new AbstractRunnable() {
+            @Override
+            public void onFailure(Throwable t) {
+                throw new RuntimeException(t);
+            }
+
+            @Override
+            protected void doRun() throws Exception {
+                r.run();
+            }
+        };
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
new file mode 100644
index 00000000000..cce349f417c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.nullValue;
+
+public class XContentParserTests extends ESTestCase {
+
+    public void testReadList() throws IOException {
+        assertThat(readList("{\"foo\": [\"bar\"]}"), contains("bar"));
+        assertThat(readList("{\"foo\": [\"bar\",\"baz\"]}"), contains("bar", "baz"));
+        assertThat(readList("{\"foo\": [1, 2, 3], \"bar\": 4}"), contains(1, 2, 3));
+        assertThat(readList("{\"foo\": [{\"bar\":1},{\"baz\":2},{\"qux\":3}]}"), hasSize(3));
+        assertThat(readList("{\"foo\": [null]}"), contains(nullValue()));
+        assertThat(readList("{\"foo\": []}"), hasSize(0));
+        assertThat(readList("{\"foo\": [1]}"), contains(1));
+        assertThat(readList("{\"foo\": [1,2]}"), contains(1, 2));
+        assertThat(readList("{\"foo\": [{},{},{},{}]}"), hasSize(4));
+    }
+
+    public void testReadListThrowsException() throws IOException {
+        // Calling XContentParser.list() or listOrderedMap() to read a simple
+        // value or object should throw an exception
+        assertReadListThrowsException("{\"foo\": \"bar\"}");
+        assertReadListThrowsException("{\"foo\": 1, \"bar\": 2}");
+        assertReadListThrowsException("{\"foo\": {\"bar\":\"baz\"}}");
+    }
+
+    @SuppressWarnings("unchecked")
+    private static <T> List<T> readList(String source) throws IOException {
+        try (XContentParser parser = XContentType.JSON.xContent().createParser(source)) {
+            XContentParser.Token token = parser.nextToken();
+            assertThat(token, equalTo(XContentParser.Token.START_OBJECT));
+            token = parser.nextToken();
+            assertThat(token, equalTo(XContentParser.Token.FIELD_NAME));
+            assertThat(parser.currentName(), equalTo("foo"));
+            return (List<T>) (randomBoolean() ? parser.listOrderedMap() : parser.list());
+        }
+    }
+
+    private void assertReadListThrowsException(String source) {
+        try {
+            readList(source);
+            fail("should have thrown a parse exception");
+        } catch (Exception e) {
+            assertThat(e, instanceOf(ElasticsearchParseException.class));
+            assertThat(e.getMessage(), containsString("Failed to parse list"));
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java
index 9129e3c05b3..36b16d6a176 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java
@@ -166,7 +166,7 @@ public class XContentBuilderTests extends ESTestCase {
 
         byte[] data = bos.bytes().toBytes();
         String sData = new String(data, "UTF8");
-        System.out.println("DATA: " + sData);
+        assertThat(sData, equalTo("{\"name\":\"something\", source : { test : \"value\" },\"name2\":\"something2\"}"));
     }
 
     public void testFieldCaseConversion() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/consistencylevel/WriteConsistencyLevelIT.java b/core/src/test/java/org/elasticsearch/consistencylevel/WriteConsistencyLevelIT.java
index b696c445f30..46c027cb91c 100644
--- a/core/src/test/java/org/elasticsearch/consistencylevel/WriteConsistencyLevelIT.java
+++ b/core/src/test/java/org/elasticsearch/consistencylevel/WriteConsistencyLevelIT.java
@@ -40,7 +40,7 @@ public class WriteConsistencyLevelIT extends ESIntegTestCase {
         prepareCreate("test", 1, Settings.settingsBuilder().put("index.number_of_shards", 1).put("index.number_of_replicas", 2)).execute().actionGet();
 
         ClusterHealthResponse clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForActiveShards(1).setWaitForYellowStatus().execute().actionGet();
-        logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
+        logger.info("Done Cluster Health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));
 
@@ -60,7 +60,7 @@ public class WriteConsistencyLevelIT extends ESIntegTestCase {
         allowNodes("test", 2);
 
         clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForActiveShards(2).setWaitForYellowStatus().execute().actionGet();
-        logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
+        logger.info("Done Cluster Health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));
 
@@ -82,7 +82,7 @@ public class WriteConsistencyLevelIT extends ESIntegTestCase {
 
         allowNodes("test", 3);
         clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForActiveShards(3).setWaitForGreenStatus().execute().actionGet();
-        logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
+        logger.info("Done Cluster Health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
 
diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
index 1728ac5ca6f..68136709771 100644
--- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
@@ -26,7 +26,6 @@ import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
@@ -40,6 +39,7 @@ import org.elasticsearch.cluster.routing.Murmur3HashFunction;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
@@ -187,15 +187,27 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
         return pluginList(MockTransportService.TestPlugin.class);
     }
 
-    private void configureUnicastCluster(int numberOfNodes, @Nullable int[] unicastHostsOrdinals, int minimumMasterNode) throws
-            ExecutionException, InterruptedException {
+    private void configureUnicastCluster(
+        int numberOfNodes,
+        @Nullable int[] unicastHostsOrdinals,
+        int minimumMasterNode
+    ) throws ExecutionException, InterruptedException {
+        configureUnicastCluster(DEFAULT_SETTINGS, numberOfNodes, unicastHostsOrdinals, minimumMasterNode);
+    }
+
+    private void configureUnicastCluster(
+        Settings settings,
+        int numberOfNodes,
+        @Nullable int[] unicastHostsOrdinals,
+        int minimumMasterNode
+    ) throws ExecutionException, InterruptedException {
         if (minimumMasterNode < 0) {
             minimumMasterNode = numberOfNodes / 2 + 1;
         }
         logger.info("---> configured unicast");
         // TODO: Rarely use default settings form some of these
         Settings nodeSettings = Settings.builder()
-                .put(DEFAULT_SETTINGS)
+                .put(settings)
                 .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minimumMasterNode)
                 .build();
 
@@ -208,7 +220,6 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
         }
     }
 
-
     /**
      * Test that no split brain occurs under partial network partition. See https://github.com/elastic/elasticsearch/issues/2488
      */
@@ -217,7 +228,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
 
         // Figure out what is the elected master node
         final String masterNode = internalCluster().getMasterName();
-        logger.info("---> legit elected master node=" + masterNode);
+        logger.info("---> legit elected master node={}", masterNode);
 
         // Pick a node that isn't the elected master.
         Set<String> nonMasters = new HashSet<>(nodes);
@@ -504,7 +515,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
             }
 
             int docsPerIndexer = randomInt(3);
-            logger.info("indexing " + docsPerIndexer + " docs per indexer before partition");
+            logger.info("indexing {} docs per indexer before partition", docsPerIndexer);
             countDownLatchRef.set(new CountDownLatch(docsPerIndexer * indexers.size()));
             for (Semaphore semaphore : semaphores) {
                 semaphore.release(docsPerIndexer);
@@ -516,7 +527,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
                 disruptionScheme.startDisrupting();
 
                 docsPerIndexer = 1 + randomInt(5);
-                logger.info("indexing " + docsPerIndexer + " docs per indexer during partition");
+                logger.info("indexing {} docs per indexer during partition", docsPerIndexer);
                 countDownLatchRef.set(new CountDownLatch(docsPerIndexer * indexers.size()));
                 Collections.shuffle(semaphores, random());
                 for (Semaphore semaphore : semaphores) {
@@ -551,11 +562,11 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
             }
         } finally {
             if (exceptedExceptions.size() > 0) {
-                StringBuilder sb = new StringBuilder("Indexing exceptions during disruption:");
+                StringBuilder sb = new StringBuilder();
                 for (Exception e : exceptedExceptions) {
                     sb.append("\n").append(e.getMessage());
                 }
-                logger.debug(sb.toString());
+                logger.debug("Indexing exceptions during disruption: {}", sb);
             }
             logger.info("shutting down indexers");
             stop.set(true);
@@ -752,7 +763,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
                 .get();
         assertThat(indexResponse.getVersion(), equalTo(1L));
 
-        logger.info("Verifying if document exists via node[" + notIsolatedNode + "]");
+        logger.info("Verifying if document exists via node[{}]", notIsolatedNode);
         GetResponse getResponse = internalCluster().client(notIsolatedNode).prepareGet("test", "type", indexResponse.getId())
                 .setPreference("_local")
                 .get();
@@ -766,7 +777,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
         ensureGreen("test");
 
         for (String node : nodes) {
-            logger.info("Verifying if document exists after isolating node[" + isolatedNode + "] via node[" + node + "]");
+            logger.info("Verifying if document exists after isolating node[{}] via node[{}]", isolatedNode, node);
             getResponse = internalCluster().client(node).prepareGet("test", "type", indexResponse.getId())
                     .setPreference("_local")
                     .get();
@@ -785,7 +796,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
         List<String> nodes = startCluster(4, -1, new int[]{0});
         // Figure out what is the elected master node
         final String masterNode = internalCluster().getMasterName();
-        logger.info("---> legit elected master node=" + masterNode);
+        logger.info("---> legit elected master node={}", masterNode);
         List<String> otherNodes = new ArrayList<>(nodes);
         otherNodes.remove(masterNode);
         otherNodes.remove(nodes.get(0)); // <-- Don't isolate the node that is in the unicast endpoint for all the other nodes.
@@ -1108,25 +1119,40 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
      * Tests that indices are properly deleted even if there is a master transition in between.
      * Test for https://github.com/elastic/elasticsearch/issues/11665
      */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/16890")
     public void testIndicesDeleted() throws Exception {
-        configureUnicastCluster(3, null, 2);
+        final Settings settings = Settings.builder()
+                                      .put(DEFAULT_SETTINGS)
+                                      .put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "0s") // don't wait on isolated data node
+                                      .put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "30s") // wait till cluster state is committed
+                                      .build();
+        final String idxName = "test";
+        configureUnicastCluster(settings, 3, null, 2);
         InternalTestCluster.Async<List<String>> masterNodes = internalCluster().startMasterOnlyNodesAsync(2);
         InternalTestCluster.Async<String> dataNode = internalCluster().startDataOnlyNodeAsync();
         dataNode.get();
-        masterNodes.get();
+        final List<String> allMasterEligibleNodes = masterNodes.get();
         ensureStableCluster(3);
         assertAcked(prepareCreate("test"));
         ensureYellow();
 
-        String masterNode1 = internalCluster().getMasterName();
+        final String masterNode1 = internalCluster().getMasterName();
         NetworkPartition networkPartition = new NetworkUnresponsivePartition(masterNode1, dataNode.get(), getRandom());
         internalCluster().setDisruptionScheme(networkPartition);
         networkPartition.startDisrupting();
-        internalCluster().client(masterNode1).admin().indices().prepareDelete("test").setTimeout("1s").get();
+        // We know this will time out due to the partition, we check manually below to not proceed until
+        // the delete has been applied to the master node and the master eligible node.
+        internalCluster().client(masterNode1).admin().indices().prepareDelete(idxName).setTimeout("0s").get();
+        // Don't restart the master node until we know the index deletion has taken effect on master and the master eligible node.
+        assertBusy(() -> {
+            for (String masterNode : allMasterEligibleNodes) {
+                final ClusterState masterState = internalCluster().clusterService(masterNode).state();
+                assertTrue("index not deleted on " + masterNode, masterState.metaData().hasIndex(idxName) == false &&
+                                                                 masterState.status() == ClusterState.ClusterStateStatus.APPLIED);
+            }
+        });
         internalCluster().restartNode(masterNode1, InternalTestCluster.EMPTY_CALLBACK);
         ensureYellow();
-        assertFalse(client().admin().indices().prepareExists("test").get().isExists());
+        assertFalse(client().admin().indices().prepareExists(idxName).get().isExists());
     }
 
     protected NetworkPartition addRandomPartition() {
diff --git a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
index 65484b81c79..b58354b77d1 100644
--- a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
@@ -24,13 +24,13 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.zen.fd.FaultDetection;
 import org.elasticsearch.discovery.zen.fd.MasterFaultDetection;
 import org.elasticsearch.discovery.zen.fd.NodesFaultDetection;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.cluster.NoopClusterService;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportConnectionListener;
@@ -44,10 +44,13 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
 import static java.util.Collections.emptyMap;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 
 public class ZenFaultDetectionTests extends ESTestCase {
     protected ThreadPool threadPool;
+    protected ClusterService clusterService;
 
     protected static final Version version0 = Version.fromId(/*0*/99);
     protected DiscoveryNode nodeA;
@@ -62,6 +65,7 @@ public class ZenFaultDetectionTests extends ESTestCase {
     public void setUp() throws Exception {
         super.setUp();
         threadPool = new ThreadPool(getClass().getName());
+        clusterService = createClusterService(threadPool);
         serviceA = build(Settings.builder().put("name", "TS_A").build(), version0);
         nodeA = new DiscoveryNode("TS_A", "TS_A", serviceA.boundAddress().publishAddress(), emptyMap(), version0);
         serviceB = build(Settings.builder().put("name", "TS_B").build(), version1);
@@ -100,13 +104,14 @@ public class ZenFaultDetectionTests extends ESTestCase {
         super.tearDown();
         serviceA.close();
         serviceB.close();
+        clusterService.close();
         terminate(threadPool);
     }
 
     protected MockTransportService build(Settings settings, Version version) {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
         MockTransportService transportService = new MockTransportService(Settings.EMPTY,
-                new LocalTransport(settings, threadPool, version, namedWriteableRegistry), threadPool, namedWriteableRegistry);
+                new LocalTransport(settings, threadPool, version, namedWriteableRegistry), threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
         return transportService;
@@ -186,21 +191,18 @@ public class ZenFaultDetectionTests extends ESTestCase {
                 .put(FaultDetection.PING_INTERVAL_SETTING.getKey(), "5m");
         ClusterName clusterName = new ClusterName(randomAsciiOfLengthBetween(3, 20));
         final ClusterState state = ClusterState.builder(clusterName).nodes(buildNodesForA(false)).build();
+        setState(clusterService, state);
         MasterFaultDetection masterFD = new MasterFaultDetection(settings.build(), threadPool, serviceA, clusterName,
-                new NoopClusterService(state));
+                clusterService);
         masterFD.start(nodeB, "test");
 
         final String[] failureReason = new String[1];
         final DiscoveryNode[] failureNode = new DiscoveryNode[1];
         final CountDownLatch notified = new CountDownLatch(1);
-        masterFD.addListener(new MasterFaultDetection.Listener() {
-
-            @Override
-            public void onMasterFailure(DiscoveryNode masterNode, Throwable cause, String reason) {
-                failureNode[0] = masterNode;
-                failureReason[0] = reason;
-                notified.countDown();
-            }
+        masterFD.addListener((masterNode, cause, reason) -> {
+            failureNode[0] = masterNode;
+            failureReason[0] = reason;
+            notified.countDown();
         });
         // will raise a disconnect on A
         serviceB.stop();
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
index 0ca261cbf65..96e7a90ece6 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
@@ -30,6 +30,7 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
@@ -40,9 +41,12 @@ import org.elasticsearch.common.util.concurrent.BaseFuture;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.discovery.zen.membership.MembershipAction;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.cluster.TestClusterService;
 import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Before;
+import org.junit.BeforeClass;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -55,28 +59,52 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
 @TestLogging("discovery.zen:TRACE")
 public class NodeJoinControllerTests extends ESTestCase {
 
-    private TestClusterService clusterService;
+    private static ThreadPool threadPool;
+
+    private ClusterService clusterService;
     private NodeJoinController nodeJoinController;
 
+    @BeforeClass
+    public static void beforeClass() {
+        threadPool = new ThreadPool("ShardReplicationTests");
+    }
+
+    @AfterClass
+    public static void afterClass() {
+        ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
+        threadPool = null;
+    }
+
     @Before
     public void setUp() throws Exception {
         super.setUp();
-        clusterService = new TestClusterService();
+        clusterService = createClusterService(threadPool);
         final DiscoveryNodes initialNodes = clusterService.state().nodes();
         final DiscoveryNode localNode = initialNodes.localNode();
         // make sure we have a master
-        clusterService.setState(ClusterState.builder(clusterService.state()).nodes(DiscoveryNodes.builder(initialNodes).masterNodeId(localNode.id())));
+        setState(clusterService, ClusterState.builder(clusterService.state()).nodes(
+                DiscoveryNodes.builder(initialNodes).masterNodeId(localNode.id())));
         nodeJoinController = new NodeJoinController(clusterService, new NoopRoutingService(Settings.EMPTY),
-                new DiscoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), Settings.EMPTY);
+                new DiscoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
+                Settings.EMPTY);
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        clusterService.close();
     }
 
     public void testSimpleJoinAccumulation() throws InterruptedException, ExecutionException {
@@ -97,21 +125,29 @@ public class NodeJoinControllerTests extends ESTestCase {
             pendingJoins.add(joinNodeAsync(node));
         }
         nodeJoinController.stopAccumulatingJoins("test");
+        boolean hadSyncJoin = false;
         for (int i = randomInt(5); i > 0; i--) {
             DiscoveryNode node = newNode(nodeId++);
             nodes.add(node);
             joinNode(node);
+            hadSyncJoin = true;
+        }
+        if (hadSyncJoin) {
+            for (Future<Void> joinFuture : pendingJoins) {
+                assertThat(joinFuture.isDone(), equalTo(true));
+            }
         }
-        assertNodesInCurrentState(nodes);
         for (Future<Void> joinFuture : pendingJoins) {
-            assertThat(joinFuture.isDone(), equalTo(true));
+            joinFuture.get();
         }
+
+        assertNodesInCurrentState(nodes);
     }
 
     public void testFailingJoinsWhenNotMaster() throws ExecutionException, InterruptedException {
         // remove current master flag
         DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterService.state().nodes()).masterNodeId(null);
-        clusterService.setState(ClusterState.builder(clusterService.state()).nodes(nodes));
+        setState(clusterService, ClusterState.builder(clusterService.state()).nodes(nodes));
         int nodeId = 0;
         try {
             joinNode(newNode(nodeId++));
@@ -142,7 +178,7 @@ public class NodeJoinControllerTests extends ESTestCase {
 
     public void testSimpleMasterElectionWithoutRequiredJoins() throws InterruptedException, ExecutionException {
         DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterService.state().nodes()).masterNodeId(null);
-        clusterService.setState(ClusterState.builder(clusterService.state()).nodes(nodes));
+        setState(clusterService, ClusterState.builder(clusterService.state()).nodes(nodes));
         int nodeId = 0;
         final int requiredJoins = 0;
         logger.debug("--> using requiredJoins [{}]", requiredJoins);
@@ -184,13 +220,13 @@ public class NodeJoinControllerTests extends ESTestCase {
         });
         masterElection.start();
 
-            logger.debug("--> requiredJoins is set to 0. verifying election finished");
-            electionFuture.get();
+        logger.debug("--> requiredJoins is set to 0. verifying election finished");
+        electionFuture.get();
     }
 
     public void testSimpleMasterElection() throws InterruptedException, ExecutionException {
         DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterService.state().nodes()).masterNodeId(null);
-        clusterService.setState(ClusterState.builder(clusterService.state()).nodes(nodes));
+        setState(clusterService, ClusterState.builder(clusterService.state()).nodes(nodes));
         int nodeId = 0;
         final int requiredJoins = 1 + randomInt(5);
         logger.debug("--> using requiredJoins [{}]", requiredJoins);
@@ -301,7 +337,7 @@ public class NodeJoinControllerTests extends ESTestCase {
 
     public void testMasterElectionTimeout() throws InterruptedException {
         DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterService.state().nodes()).masterNodeId(null);
-        clusterService.setState(ClusterState.builder(clusterService.state()).nodes(nodes));
+        setState(clusterService, ClusterState.builder(clusterService.state()).nodes(nodes));
         int nodeId = 0;
         final int requiredJoins = 1 + randomInt(5);
         logger.debug("--> using requiredJoins [{}]", requiredJoins);
@@ -367,7 +403,7 @@ public class NodeJoinControllerTests extends ESTestCase {
         final DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(state.nodes());
         final DiscoveryNode other_node = new DiscoveryNode("other_node", DummyTransportAddress.INSTANCE, Version.CURRENT);
         nodesBuilder.put(other_node);
-        clusterService.setState(ClusterState.builder(state).nodes(nodesBuilder));
+        setState(clusterService, ClusterState.builder(state).nodes(nodesBuilder));
 
         state = clusterService.state();
         joinNode(other_node);
@@ -413,7 +449,7 @@ public class NodeJoinControllerTests extends ESTestCase {
 
     public void testElectionWithConcurrentJoins() throws InterruptedException, BrokenBarrierException {
         DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(clusterService.state().nodes()).masterNodeId(null);
-        clusterService.setState(ClusterState.builder(clusterService.state()).nodes(nodesBuilder));
+        setState(clusterService, ClusterState.builder(clusterService.state()).nodes(nodesBuilder));
 
         nodeJoinController.startAccumulatingJoins();
 
@@ -492,7 +528,7 @@ public class NodeJoinControllerTests extends ESTestCase {
     static class NoopAllocationService extends AllocationService {
 
         public NoopAllocationService(Settings settings) {
-            super(settings, null, null, null);
+            super(settings, null, null, null, null);
         }
 
         @Override
@@ -512,11 +548,13 @@ public class NodeJoinControllerTests extends ESTestCase {
     }
 
     protected void assertNodesInCurrentState(List<DiscoveryNode> expectedNodes) {
-        DiscoveryNodes discoveryNodes = clusterService.state().nodes();
-        assertThat(discoveryNodes.prettyPrint() + "\nexpected: " + expectedNodes.toString(), discoveryNodes.size(), equalTo(expectedNodes.size()));
+        final ClusterState state = clusterService.state();
+        logger.info("assert for [{}] in:\n{}", expectedNodes, state.prettyPrint());
+        DiscoveryNodes discoveryNodes = state.nodes();
         for (DiscoveryNode node : expectedNodes) {
             assertThat("missing " + node + "\n" + discoveryNodes.prettyPrint(), discoveryNodes.get(node.id()), equalTo(node));
         }
+        assertThat(discoveryNodes.size(), equalTo(expectedNodes.size()));
     }
 
     static class SimpleFuture extends BaseFuture<Void> {
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
index 9ad10cc3888..d14bd583c44 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
@@ -25,12 +25,12 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
@@ -275,7 +275,7 @@ public class ZenDiscoveryIT extends ESIntegTestCase {
         Settings nodeSettings = Settings.settingsBuilder()
                 .put("discovery.type", "zen") // <-- To override the local setting if set externally
                 .build();
-        String nodeName = internalCluster().startNode(nodeSettings, Version.V_5_0_0);
+        String nodeName = internalCluster().startNode(nodeSettings, Version.V_5_0_0_alpha1);
         ZenDiscovery zenDiscovery = (ZenDiscovery) internalCluster().getInstance(Discovery.class, nodeName);
         ClusterService clusterService = internalCluster().getInstance(ClusterService.class, nodeName);
         DiscoveryNode node = new DiscoveryNode("_node_id", new InetSocketTransportAddress(InetAddress.getByName("0.0.0.0"), 0), Version.V_2_0_0);
@@ -292,13 +292,13 @@ public class ZenDiscoveryIT extends ESIntegTestCase {
         });
 
         assertThat(holder.get(), notNullValue());
-        assertThat(holder.get().getMessage(), equalTo("Can't handle join request from a node with a version [2.0.0] that is lower than the minimum compatible version [" + Version.V_5_0_0.minimumCompatibilityVersion() + "]"));
+        assertThat(holder.get().getMessage(), equalTo("Can't handle join request from a node with a version [2.0.0] that is lower than the minimum compatible version [" + Version.V_5_0_0_alpha1.minimumCompatibilityVersion() + "]"));
     }
 
     public void testJoinElectedMaster_incompatibleMinVersion() {
-        ElectMasterService electMasterService = new ElectMasterService(Settings.EMPTY, Version.V_5_0_0);
+        ElectMasterService electMasterService = new ElectMasterService(Settings.EMPTY, Version.V_5_0_0_alpha1);
 
-        DiscoveryNode node = new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), Version.V_5_0_0);
+        DiscoveryNode node = new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), Version.V_5_0_0_alpha1);
         assertThat(electMasterService.electMaster(Collections.singletonList(node)), sameInstance(node));
         node = new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), Version.V_2_0_0);
         assertThat("Can't join master because version 2.0.0 is lower than the minimum compatable version 5.0.0 can support", electMasterService.electMaster(Collections.singletonList(node)), nullValue());
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java
index e1d6beecaa9..468d01aaecb 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java
@@ -25,10 +25,19 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.discovery.zen.ping.ZenPing;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
 import static org.elasticsearch.discovery.zen.ZenDiscovery.shouldIgnoreOrRejectNewClusterState;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
 /**
  */
@@ -89,4 +98,34 @@ public class ZenDiscoveryUnitTests extends ESTestCase {
         }
         assertFalse("should not ignore, because current state doesn't have a master", shouldIgnoreOrRejectNewClusterState(logger, currentState.build(), newState.build()));
     }
+
+    public void testFilterNonMasterPingResponse() {
+        ArrayList<ZenPing.PingResponse> responses = new ArrayList<>();
+        ArrayList<DiscoveryNode> masterNodes = new ArrayList<>();
+        ArrayList<DiscoveryNode> allNodes = new ArrayList<>();
+        for (int i = randomIntBetween(10, 20); i >= 0; i--) {
+            Map<String, String> attrs = new HashMap<>();
+            for (String attr : randomSubsetOf(
+                    Arrays.asList(DiscoveryNode.INGEST_ATTR, DiscoveryNode.DATA_ATTR, DiscoveryNode.MASTER_ATTR))) {
+                attrs.put(attr, randomBoolean() + "");
+            }
+
+            DiscoveryNode node = new DiscoveryNode("node_" + i, "id_" + i, DummyTransportAddress.INSTANCE, attrs, Version.CURRENT);
+            responses.add(new ZenPing.PingResponse(node, randomBoolean() ? null : node, new ClusterName("test"), randomBoolean()));
+            allNodes.add(node);
+            if (node.isMasterNode()) {
+                masterNodes.add(node);
+            }
+        }
+
+        boolean ignore = randomBoolean();
+        List<ZenPing.PingResponse> filtered = ZenDiscovery.filterPingResponses(
+                responses.toArray(new ZenPing.PingResponse[responses.size()]), ignore, logger);
+        final List<DiscoveryNode> filteredNodes = filtered.stream().map(ZenPing.PingResponse::node).collect(Collectors.toList());
+        if (ignore) {
+            assertThat(filteredNodes, equalTo(masterNodes));
+        } else {
+            assertThat(filteredNodes, equalTo(allNodes));
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
index d255a80fbb8..3c13351a125 100644
--- a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
+++ b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.gateway.MetaDataStateFormat;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.ShardId;
@@ -36,7 +37,11 @@ import org.elasticsearch.test.IndexSettingsModule;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -129,21 +134,22 @@ public class NodeEnvironmentTests extends ESTestCase {
     public void testShardLock() throws IOException {
         final NodeEnvironment env = newNodeEnvironment();
 
-        ShardLock fooLock = env.shardLock(new ShardId("foo", "_na_", 0));
-        assertEquals(new ShardId("foo", "_na_", 0), fooLock.getShardId());
+        Index index = new Index("foo", "fooUUID");
+        ShardLock fooLock = env.shardLock(new ShardId(index, 0));
+        assertEquals(new ShardId(index, 0), fooLock.getShardId());
 
         try {
-            env.shardLock(new ShardId("foo", "_na_", 0));
+            env.shardLock(new ShardId(index, 0));
             fail("shard is locked");
         } catch (LockObtainFailedException ex) {
             // expected
         }
-        for (Path path : env.indexPaths("foo")) {
+        for (Path path : env.indexPaths(index)) {
             Files.createDirectories(path.resolve("0"));
             Files.createDirectories(path.resolve("1"));
         }
         try {
-            env.lockAllForIndex(new Index("foo", "_na_"), idxSettings, randomIntBetween(0, 10));
+            env.lockAllForIndex(index, idxSettings, randomIntBetween(0, 10));
             fail("shard 0 is locked");
         } catch (LockObtainFailedException ex) {
             // expected
@@ -151,11 +157,11 @@ public class NodeEnvironmentTests extends ESTestCase {
 
         fooLock.close();
         // can lock again?
-        env.shardLock(new ShardId("foo", "_na_", 0)).close();
+        env.shardLock(new ShardId(index, 0)).close();
 
-        List<ShardLock> locks = env.lockAllForIndex(new Index("foo", "_na_"), idxSettings, randomIntBetween(0, 10));
+        List<ShardLock> locks = env.lockAllForIndex(index, idxSettings, randomIntBetween(0, 10));
         try {
-            env.shardLock(new ShardId("foo", "_na_", 0));
+            env.shardLock(new ShardId(index, 0));
             fail("shard is locked");
         } catch (LockObtainFailedException ex) {
             // expected
@@ -165,18 +171,45 @@ public class NodeEnvironmentTests extends ESTestCase {
         env.close();
     }
 
-    public void testGetAllIndices() throws Exception {
+    public void testAvailableIndexFolders() throws Exception {
         final NodeEnvironment env = newNodeEnvironment();
         final int numIndices = randomIntBetween(1, 10);
+        Set<String> actualPaths = new HashSet<>();
         for (int i = 0; i < numIndices; i++) {
-            for (Path path : env.indexPaths("foo" + i)) {
-                Files.createDirectories(path);
+            Index index = new Index("foo" + i, "fooUUID" + i);
+            for (Path path : env.indexPaths(index)) {
+                Files.createDirectories(path.resolve(MetaDataStateFormat.STATE_DIR_NAME));
+                actualPaths.add(path.getFileName().toString());
             }
         }
-        Set<String> indices = env.findAllIndices();
-        assertEquals(indices.size(), numIndices);
+
+        assertThat(actualPaths, equalTo(env.availableIndexFolders()));
+        assertTrue("LockedShards: " + env.lockedShards(), env.lockedShards().isEmpty());
+        env.close();
+    }
+
+    public void testResolveIndexFolders() throws Exception {
+        final NodeEnvironment env = newNodeEnvironment();
+        final int numIndices = randomIntBetween(1, 10);
+        Map<String, List<Path>> actualIndexDataPaths = new HashMap<>();
         for (int i = 0; i < numIndices; i++) {
-            assertTrue(indices.contains("foo" + i));
+            Index index = new Index("foo" + i, "fooUUID" + i);
+            Path[] indexPaths = env.indexPaths(index);
+            for (Path path : indexPaths) {
+                Files.createDirectories(path);
+                String fileName = path.getFileName().toString();
+                List<Path> paths = actualIndexDataPaths.get(fileName);
+                if (paths == null) {
+                    paths = new ArrayList<>();
+                }
+                paths.add(path);
+                actualIndexDataPaths.put(fileName, paths);
+            }
+        }
+        for (Map.Entry<String, List<Path>> actualIndexDataPathEntry : actualIndexDataPaths.entrySet()) {
+            List<Path> actual = actualIndexDataPathEntry.getValue();
+            Path[] actualPaths = actual.toArray(new Path[actual.size()]);
+            assertThat(actualPaths, equalTo(env.resolveIndexFolder(actualIndexDataPathEntry.getKey())));
         }
         assertTrue("LockedShards: " + env.lockedShards(), env.lockedShards().isEmpty());
         env.close();
@@ -184,44 +217,45 @@ public class NodeEnvironmentTests extends ESTestCase {
 
     public void testDeleteSafe() throws IOException, InterruptedException {
         final NodeEnvironment env = newNodeEnvironment();
-        ShardLock fooLock = env.shardLock(new ShardId("foo", "_na_", 0));
-        assertEquals(new ShardId("foo", "_na_", 0), fooLock.getShardId());
+        final Index index = new Index("foo", "fooUUID");
+        ShardLock fooLock = env.shardLock(new ShardId(index, 0));
+        assertEquals(new ShardId(index, 0), fooLock.getShardId());
 
 
-        for (Path path : env.indexPaths("foo")) {
+        for (Path path : env.indexPaths(index)) {
             Files.createDirectories(path.resolve("0"));
             Files.createDirectories(path.resolve("1"));
         }
 
         try {
-            env.deleteShardDirectorySafe(new ShardId("foo", "_na_", 0), idxSettings);
+            env.deleteShardDirectorySafe(new ShardId(index, 0), idxSettings);
             fail("shard is locked");
         } catch (LockObtainFailedException ex) {
             // expected
         }
 
-        for (Path path : env.indexPaths("foo")) {
+        for (Path path : env.indexPaths(index)) {
             assertTrue(Files.exists(path.resolve("0")));
             assertTrue(Files.exists(path.resolve("1")));
 
         }
 
-        env.deleteShardDirectorySafe(new ShardId("foo", "_na_", 1), idxSettings);
+        env.deleteShardDirectorySafe(new ShardId(index, 1), idxSettings);
 
-        for (Path path : env.indexPaths("foo")) {
+        for (Path path : env.indexPaths(index)) {
             assertTrue(Files.exists(path.resolve("0")));
             assertFalse(Files.exists(path.resolve("1")));
         }
 
         try {
-            env.deleteIndexDirectorySafe(new Index("foo", "_na_"), randomIntBetween(0, 10), idxSettings);
+            env.deleteIndexDirectorySafe(index, randomIntBetween(0, 10), idxSettings);
             fail("shard is locked");
         } catch (LockObtainFailedException ex) {
             // expected
         }
         fooLock.close();
 
-        for (Path path : env.indexPaths("foo")) {
+        for (Path path : env.indexPaths(index)) {
             assertTrue(Files.exists(path));
         }
 
@@ -242,7 +276,7 @@ public class NodeEnvironmentTests extends ESTestCase {
                 @Override
                 protected void doRun() throws Exception {
                     start.await();
-                    try (ShardLock autoCloses = env.shardLock(new ShardId("foo", "_na_", 0))) {
+                    try (ShardLock autoCloses = env.shardLock(new ShardId(index, 0))) {
                         blockLatch.countDown();
                         Thread.sleep(randomIntBetween(1, 10));
                     }
@@ -257,11 +291,11 @@ public class NodeEnvironmentTests extends ESTestCase {
         start.countDown();
         blockLatch.await();
 
-        env.deleteIndexDirectorySafe(new Index("foo", "_na_"), 5000, idxSettings);
+        env.deleteIndexDirectorySafe(index, 5000, idxSettings);
 
         assertNull(threadException.get());
 
-        for (Path path : env.indexPaths("foo")) {
+        for (Path path : env.indexPaths(index)) {
             assertFalse(Files.exists(path));
         }
         latch.await();
@@ -300,7 +334,7 @@ public class NodeEnvironmentTests extends ESTestCase {
                     for (int i = 0; i < iters; i++) {
                         int shard = randomIntBetween(0, counts.length - 1);
                         try {
-                            try (ShardLock autoCloses = env.shardLock(new ShardId("foo", "_na_", shard), scaledRandomIntBetween(0, 10))) {
+                            try (ShardLock autoCloses = env.shardLock(new ShardId("foo", "fooUUID", shard), scaledRandomIntBetween(0, 10))) {
                                 counts[shard].value++;
                                 countsAtomic[shard].incrementAndGet();
                                 assertEquals(flipFlop[shard].incrementAndGet(), 1);
@@ -334,37 +368,38 @@ public class NodeEnvironmentTests extends ESTestCase {
         String[] dataPaths = tmpPaths();
         NodeEnvironment env = newNodeEnvironment(dataPaths, "/tmp", Settings.EMPTY);
 
-        IndexSettings s1 = IndexSettingsModule.newIndexSettings("myindex", Settings.EMPTY);
-        IndexSettings s2 = IndexSettingsModule.newIndexSettings("myindex", Settings.builder().put(IndexMetaData.SETTING_DATA_PATH, "/tmp/foo").build());
-        Index index = new Index("myindex", "_na_");
+        final Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_INDEX_UUID, "myindexUUID").build();
+        IndexSettings s1 = IndexSettingsModule.newIndexSettings("myindex", indexSettings);
+        IndexSettings s2 = IndexSettingsModule.newIndexSettings("myindex", Settings.builder().put(indexSettings).put(IndexMetaData.SETTING_DATA_PATH, "/tmp/foo").build());
+        Index index = new Index("myindex", "myindexUUID");
         ShardId sid = new ShardId(index, 0);
 
         assertFalse("no settings should mean no custom data path", s1.hasCustomDataPath());
         assertTrue("settings with path_data should have a custom data path", s2.hasCustomDataPath());
 
         assertThat(env.availableShardPaths(sid), equalTo(env.availableShardPaths(sid)));
-        assertThat(env.resolveCustomLocation(s2, sid), equalTo(PathUtils.get("/tmp/foo/0/myindex/0")));
+        assertThat(env.resolveCustomLocation(s2, sid), equalTo(PathUtils.get("/tmp/foo/0/" + index.getUUID() + "/0")));
 
         assertThat("shard paths with a custom data_path should contain only regular paths",
                 env.availableShardPaths(sid),
-                equalTo(stringsToPaths(dataPaths, "elasticsearch/nodes/0/indices/myindex/0")));
+                equalTo(stringsToPaths(dataPaths, "elasticsearch/nodes/0/indices/" + index.getUUID() + "/0")));
 
         assertThat("index paths uses the regular template",
-                env.indexPaths(index.getName()), equalTo(stringsToPaths(dataPaths, "elasticsearch/nodes/0/indices/myindex")));
+                env.indexPaths(index), equalTo(stringsToPaths(dataPaths, "elasticsearch/nodes/0/indices/" + index.getUUID())));
 
         env.close();
         NodeEnvironment env2 = newNodeEnvironment(dataPaths, "/tmp",
                 Settings.builder().put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), false).build());
 
         assertThat(env2.availableShardPaths(sid), equalTo(env2.availableShardPaths(sid)));
-        assertThat(env2.resolveCustomLocation(s2, sid), equalTo(PathUtils.get("/tmp/foo/myindex/0")));
+        assertThat(env2.resolveCustomLocation(s2, sid), equalTo(PathUtils.get("/tmp/foo/" + index.getUUID() + "/0")));
 
         assertThat("shard paths with a custom data_path should contain only regular paths",
                 env2.availableShardPaths(sid),
-                equalTo(stringsToPaths(dataPaths, "elasticsearch/nodes/0/indices/myindex/0")));
+                equalTo(stringsToPaths(dataPaths, "elasticsearch/nodes/0/indices/" + index.getUUID() + "/0")));
 
         assertThat("index paths uses the regular template",
-                env2.indexPaths(index.getName()), equalTo(stringsToPaths(dataPaths, "elasticsearch/nodes/0/indices/myindex")));
+                env2.indexPaths(index), equalTo(stringsToPaths(dataPaths, "elasticsearch/nodes/0/indices/" + index.getUUID())));
 
         env2.close();
     }
diff --git a/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java b/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java
index 95c52f89933..51536375dca 100644
--- a/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java
@@ -29,6 +29,7 @@ import org.hamcrest.Matchers;
 
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -53,6 +54,47 @@ public class DanglingIndicesStateTests extends ESTestCase {
             assertTrue(danglingState.getDanglingIndices().isEmpty());
         }
     }
+    public void testDanglingIndicesDiscovery() throws Exception {
+        try (NodeEnvironment env = newNodeEnvironment()) {
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            DanglingIndicesState danglingState = new DanglingIndicesState(Settings.EMPTY, env, metaStateService, null);
+
+            assertTrue(danglingState.getDanglingIndices().isEmpty());
+            MetaData metaData = MetaData.builder().build();
+            final Settings.Builder settings = Settings.builder().put(indexSettings).put(IndexMetaData.SETTING_INDEX_UUID, "test1UUID");
+            IndexMetaData dangledIndex = IndexMetaData.builder("test1").settings(settings).build();
+            metaStateService.writeIndex("test_write", dangledIndex);
+            Map<Index, IndexMetaData> newDanglingIndices = danglingState.findNewDanglingIndices(metaData);
+            assertTrue(newDanglingIndices.containsKey(dangledIndex.getIndex()));
+            metaData = MetaData.builder().put(dangledIndex, false).build();
+            newDanglingIndices = danglingState.findNewDanglingIndices(metaData);
+            assertFalse(newDanglingIndices.containsKey(dangledIndex.getIndex()));
+        }
+    }
+
+    public void testInvalidIndexFolder() throws Exception {
+        try (NodeEnvironment env = newNodeEnvironment()) {
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            DanglingIndicesState danglingState = new DanglingIndicesState(Settings.EMPTY, env, metaStateService, null);
+
+            MetaData metaData = MetaData.builder().build();
+            final String uuid = "test1UUID";
+            final Settings.Builder settings = Settings.builder().put(indexSettings).put(IndexMetaData.SETTING_INDEX_UUID, uuid);
+            IndexMetaData dangledIndex = IndexMetaData.builder("test1").settings(settings).build();
+            metaStateService.writeIndex("test_write", dangledIndex);
+            for (Path path : env.resolveIndexFolder(uuid)) {
+                if (Files.exists(path)) {
+                    Files.move(path, path.resolveSibling("invalidUUID"), StandardCopyOption.ATOMIC_MOVE);
+                }
+            }
+            try {
+                danglingState.findNewDanglingIndices(metaData);
+                fail("no exception thrown for invalid folder name");
+            } catch (IllegalStateException e) {
+                assertThat(e.getMessage(), equalTo("[invalidUUID] invalid index folder name, rename to [test1UUID]"));
+            }
+        }
+    }
 
     public void testDanglingProcessing() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
@@ -61,15 +103,16 @@ public class DanglingIndicesStateTests extends ESTestCase {
 
             MetaData metaData = MetaData.builder().build();
 
-            IndexMetaData dangledIndex = IndexMetaData.builder("test1").settings(indexSettings).build();
-            metaStateService.writeIndex("test_write", dangledIndex, null);
+            final Settings.Builder settings = Settings.builder().put(indexSettings).put(IndexMetaData.SETTING_INDEX_UUID, "test1UUID");
+            IndexMetaData dangledIndex = IndexMetaData.builder("test1").settings(settings).build();
+            metaStateService.writeIndex("test_write", dangledIndex);
 
             // check that several runs when not in the metadata still keep the dangled index around
             int numberOfChecks = randomIntBetween(1, 10);
             for (int i = 0; i < numberOfChecks; i++) {
-                Map<String, IndexMetaData> newDanglingIndices = danglingState.findNewDanglingIndices(metaData);
+                Map<Index, IndexMetaData> newDanglingIndices = danglingState.findNewDanglingIndices(metaData);
                 assertThat(newDanglingIndices.size(), equalTo(1));
-                assertThat(newDanglingIndices.keySet(), Matchers.hasItems("test1"));
+                assertThat(newDanglingIndices.keySet(), Matchers.hasItems(dangledIndex.getIndex()));
                 assertTrue(danglingState.getDanglingIndices().isEmpty());
             }
 
@@ -77,7 +120,7 @@ public class DanglingIndicesStateTests extends ESTestCase {
                 danglingState.findNewAndAddDanglingIndices(metaData);
 
                 assertThat(danglingState.getDanglingIndices().size(), equalTo(1));
-                assertThat(danglingState.getDanglingIndices().keySet(), Matchers.hasItems("test1"));
+                assertThat(danglingState.getDanglingIndices().keySet(), Matchers.hasItems(dangledIndex.getIndex()));
             }
 
             // simulate allocation to the metadata
@@ -85,35 +128,15 @@ public class DanglingIndicesStateTests extends ESTestCase {
 
             // check that several runs when in the metadata, but not cleaned yet, still keeps dangled
             for (int i = 0; i < numberOfChecks; i++) {
-                Map<String, IndexMetaData> newDanglingIndices = danglingState.findNewDanglingIndices(metaData);
+                Map<Index, IndexMetaData> newDanglingIndices = danglingState.findNewDanglingIndices(metaData);
                 assertTrue(newDanglingIndices.isEmpty());
 
                 assertThat(danglingState.getDanglingIndices().size(), equalTo(1));
-                assertThat(danglingState.getDanglingIndices().keySet(), Matchers.hasItems("test1"));
+                assertThat(danglingState.getDanglingIndices().keySet(), Matchers.hasItems(dangledIndex.getIndex()));
             }
 
             danglingState.cleanupAllocatedDangledIndices(metaData);
             assertTrue(danglingState.getDanglingIndices().isEmpty());
         }
     }
-
-    public void testRenameOfIndexState() throws Exception {
-        try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
-            DanglingIndicesState danglingState = new DanglingIndicesState(Settings.EMPTY, env, metaStateService, null);
-
-            MetaData metaData = MetaData.builder().build();
-
-            IndexMetaData dangledIndex = IndexMetaData.builder("test1").settings(indexSettings).build();
-            metaStateService.writeIndex("test_write", dangledIndex, null);
-
-            for (Path path : env.indexPaths("test1")) {
-                Files.move(path, path.getParent().resolve("test1_renamed"));
-            }
-
-            Map<String, IndexMetaData> newDanglingIndices = danglingState.findNewDanglingIndices(metaData);
-            assertThat(newDanglingIndices.size(), equalTo(1));
-            assertThat(newDanglingIndices.keySet(), Matchers.hasItems("test1_renamed"));
-        }
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
index d0d9b227948..55955ba269d 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
@@ -19,11 +19,16 @@
 
 package org.elasticsearch.gateway;
 
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.client.Requests;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.logging.ESLogger;
@@ -31,15 +36,23 @@ import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.discovery.zen.elect.ElectMasterService;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.indices.IndexClosedException;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.InternalTestCluster.RestartCallback;
 
+import java.io.IOException;
+
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.equalTo;
@@ -56,14 +69,16 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
 
         logger.info("--> creating test index, with meta routing");
         client().admin().indices().prepareCreate("test")
-                .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("_routing").field("required", true).endObject().endObject().endObject())
+                .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("_routing")
+                    .field("required", true).endObject().endObject().endObject())
                 .execute().actionGet();
 
         logger.info("--> waiting for yellow status");
         ensureYellow();
 
         logger.info("--> verify meta _routing required exists");
-        MappingMetaData mappingMd = client().admin().cluster().prepareState().execute().actionGet().getState().metaData().index("test").mapping("type1");
+        MappingMetaData mappingMd = client().admin().cluster().prepareState().execute().actionGet().getState().metaData()
+            .index("test").mapping("type1");
         assertThat(mappingMd.routing().required(), equalTo(true));
 
         logger.info("--> restarting nodes...");
@@ -92,7 +107,8 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         ClusterStateResponse stateResponse = client().admin().cluster().prepareState().execute().actionGet();
         assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.OPEN));
         assertThat(stateResponse.getState().routingTable().index("test").shards().size(), equalTo(test.numPrimaries));
-        assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(test.totalNumShards));
+        assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(),
+            equalTo(test.totalNumShards));
 
         logger.info("--> indexing a simple document");
         client().prepareIndex("test", "type1", "1").setSource("field1", "value1").get();
@@ -129,7 +145,8 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         stateResponse = client().admin().cluster().prepareState().execute().actionGet();
         assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.OPEN));
         assertThat(stateResponse.getState().routingTable().index("test").shards().size(), equalTo(test.numPrimaries));
-        assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(test.totalNumShards));
+        assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(),
+            equalTo(test.totalNumShards));
 
         logger.info("--> trying to get the indexed document on the first index");
         GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
@@ -167,7 +184,8 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         stateResponse = client().admin().cluster().prepareState().execute().actionGet();
         assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.OPEN));
         assertThat(stateResponse.getState().routingTable().index("test").shards().size(), equalTo(test.numPrimaries));
-        assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(test.totalNumShards));
+        assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(),
+            equalTo(test.totalNumShards));
 
         logger.info("--> trying to get the indexed document on the first round (before close and shutdown)");
         getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
@@ -193,7 +211,8 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), false).build());
 
         logger.info("--> waiting for test index to be created");
-        ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setIndices("test").execute().actionGet();
+        ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setIndices("test")
+            .execute().actionGet();
         assertThat(health.isTimedOut(), equalTo(false));
 
         logger.info("--> verify we have an index");
@@ -227,7 +246,8 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefresh(true).execute().actionGet();
 
         logger.info("--> waiting for green status");
-        ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("2").execute().actionGet();
+        ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus()
+            .setWaitForNodes("2").execute().actionGet();
         assertThat(health.isTimedOut(), equalTo(false));
 
         logger.info("--> verify 1 doc in the index");
@@ -246,7 +266,8 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         client().admin().indices().prepareOpen("test").execute().actionGet();
 
         logger.info("--> waiting for green status");
-        health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("2").execute().actionGet();
+        health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes("2")
+            .execute().actionGet();
         assertThat(health.isTimedOut(), equalTo(false));
 
         logger.info("--> verify 1 doc in the index");
@@ -291,7 +312,8 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         ensureGreen();
 
         // make sure that any other events were processed
-        assertFalse(client().admin().cluster().prepareHealth().setWaitForRelocatingShards(0).setWaitForEvents(Priority.LANGUID).get().isTimedOut());
+        assertFalse(client().admin().cluster().prepareHealth().setWaitForRelocatingShards(0).setWaitForEvents(Priority.LANGUID).get()
+            .isTimedOut());
 
         logger.info("--> verify we read the right thing through alias");
         assertThat(client().prepareGet("test", "type1", "2").execute().actionGet().isExists(), equalTo(true));
@@ -364,4 +386,163 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         logger.info("--> verify the doc is there");
         assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(true));
     }
+
+    /**
+     * This test really tests worst case scenario where we have a broken setting or any setting that prevents an index from being
+     * allocated in our metadata that we recover. In that case we now have the ability to check the index on local recovery from disk
+     * if it is sane and if we can successfully create an IndexService. This also includes plugins etc.
+     */
+    public void testRecoverBrokenIndexMetadata() throws Exception {
+        logger.info("--> starting one node");
+        internalCluster().startNode();
+        logger.info("--> indexing a simple document");
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefresh(true).execute().actionGet();
+        logger.info("--> waiting for green status");
+        if (usually()) {
+            ensureYellow();
+        } else {
+            internalCluster().startNode();
+            client().admin().cluster()
+                .health(Requests.clusterHealthRequest()
+                    .waitForGreenStatus()
+                    .waitForEvents(Priority.LANGUID)
+                    .waitForRelocatingShards(0).waitForNodes("2")).actionGet();
+        }
+        ClusterState state = client().admin().cluster().prepareState().get().getState();
+        IndexMetaData metaData = state.getMetaData().index("test");
+        for (NodeEnvironment services : internalCluster().getInstances(NodeEnvironment.class)) {
+            IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(Settings.builder().put(metaData.getSettings())
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_0_0_beta1.id)
+                 // this is invalid but should be archived
+                .put("index.similarity.BM25.type", "classic")
+                 // this one is not validated ahead of time and breaks allocation
+                .put("index.analysis.filter.myCollator.type", "icu_collation")
+            ).build();
+            IndexMetaData.FORMAT.write(brokenMeta, brokenMeta.getVersion(), services.indexPaths(brokenMeta.getIndex()));
+        }
+        internalCluster().fullRestart();
+        // ensureGreen(closedIndex) waits for the index to show up in the metadata
+        // this is crucial otherwise the state call below might not contain the index yet
+        ensureGreen(metaData.getIndex().getName());
+        state = client().admin().cluster().prepareState().get().getState();
+        assertEquals(IndexMetaData.State.CLOSE, state.getMetaData().index(metaData.getIndex()).getState());
+        assertEquals("classic", state.getMetaData().index(metaData.getIndex()).getSettings().get("archived.index.similarity.BM25.type"));
+        // try to open it with the broken setting - fail again!
+        ElasticsearchException ex = expectThrows(ElasticsearchException.class, () -> client().admin().indices().prepareOpen("test").get());
+        assertEquals(ex.getMessage(), "Failed to verify index " + metaData.getIndex());
+        assertNotNull(ex.getCause());
+        assertEquals(IllegalArgumentException.class, ex.getCause().getClass());
+        assertEquals(ex.getCause().getMessage(), "Unknown tokenfilter type [icu_collation] for [myCollator]");
+
+        client().admin().indices().prepareUpdateSettings()
+            .setSettings(Settings.builder().putNull("index.analysis.filter.myCollator.type")).get();
+        client().admin().indices().prepareOpen("test").get();
+        ensureYellow();
+        logger.info("--> verify 1 doc in the index");
+        assertHitCount(client().prepareSearch().setQuery(matchAllQuery()).get(), 1L);
+    }
+
+    /**
+     * This test really tests worst case scenario where we have a missing analyzer setting.
+     * In that case we now have the ability to check the index on local recovery from disk
+     * if it is sane and if we can successfully create an IndexService.
+     * This also includes plugins etc.
+     */
+    public void testRecoverMissingAnalyzer() throws Exception {
+        logger.info("--> starting one node");
+        internalCluster().startNode();
+        prepareCreate("test").setSettings(Settings.builder()
+            .put("index.analysis.analyzer.test.tokenizer", "keyword")
+            .put("index.number_of_shards", "1"))
+            .addMapping("type1", "{\n" +
+                "    \"type1\": {\n" +
+                "      \"properties\": {\n" +
+                "        \"field1\": {\n" +
+                "          \"type\": \"text\",\n" +
+                "          \"analyzer\": \"test\"\n" +
+                "        }\n" +
+                "      }\n" +
+                "    }\n" +
+                "  }}").get();
+        logger.info("--> indexing a simple document");
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value one").setRefresh(true).execute().actionGet();
+        logger.info("--> waiting for green status");
+        if (usually()) {
+            ensureYellow();
+        } else {
+            internalCluster().startNode();
+            client().admin().cluster()
+                .health(Requests.clusterHealthRequest()
+                    .waitForGreenStatus()
+                    .waitForEvents(Priority.LANGUID)
+                    .waitForRelocatingShards(0).waitForNodes("2")).actionGet();
+        }
+        ClusterState state = client().admin().cluster().prepareState().get().getState();
+        IndexMetaData metaData = state.getMetaData().index("test");
+        for (NodeEnvironment services : internalCluster().getInstances(NodeEnvironment.class)) {
+            IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(metaData.getSettings()
+                .filter((s) -> "index.analysis.analyzer.test.tokenizer".equals(s) == false)).build();
+            IndexMetaData.FORMAT.write(brokenMeta, brokenMeta.getVersion(), services.indexPaths(brokenMeta.getIndex()));
+        }
+        internalCluster().fullRestart();
+        // ensureGreen(closedIndex) waits for the index to show up in the metadata
+        // this is crucial otherwise the state call below might not contain the index yet
+        ensureGreen(metaData.getIndex().getName());
+        state = client().admin().cluster().prepareState().get().getState();
+        assertEquals(IndexMetaData.State.CLOSE, state.getMetaData().index(metaData.getIndex()).getState());
+
+        // try to open it with the broken setting - fail again!
+        ElasticsearchException ex = expectThrows(ElasticsearchException.class, () -> client().admin().indices().prepareOpen("test").get());
+        assertEquals(ex.getMessage(), "Failed to verify index " + metaData.getIndex());
+        assertNotNull(ex.getCause());
+        assertEquals(MapperParsingException.class, ex.getCause().getClass());
+        assertEquals(ex.getCause().getMessage(), "analyzer [test] not found for field [field1]");
+
+        client().admin().indices().prepareUpdateSettings()
+            .setSettings(Settings.builder().put("index.analysis.analyzer.test.tokenizer", "keyword")).get();
+        client().admin().indices().prepareOpen("test").get();
+        ensureYellow();
+        logger.info("--> verify 1 doc in the index");
+        assertHitCount(client().prepareSearch().setQuery(matchQuery("field1", "value one")).get(), 1L);
+    }
+
+    public void testArchiveBrokenClusterSettings() throws Exception {
+        logger.info("--> starting one node");
+        internalCluster().startNode();
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefresh(true).execute().actionGet();
+        logger.info("--> waiting for green status");
+        if (usually()) {
+            ensureYellow();
+        } else {
+            internalCluster().startNode();
+            client().admin().cluster()
+                .health(Requests.clusterHealthRequest()
+                    .waitForGreenStatus()
+                    .waitForEvents(Priority.LANGUID)
+                    .waitForRelocatingShards(0).waitForNodes("2")).actionGet();
+        }
+        ClusterState state = client().admin().cluster().prepareState().get().getState();
+        MetaData metaData = state.getMetaData();
+        for (NodeEnvironment nodeEnv : internalCluster().getInstances(NodeEnvironment.class)) {
+            MetaData brokenMeta = MetaData.builder(metaData).persistentSettings(Settings.builder()
+                .put(metaData.persistentSettings()).put("this.is.unknown", true)
+                .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), "broken").build()).build();
+            MetaData.FORMAT.write(brokenMeta, metaData.version(), nodeEnv.nodeDataPaths());
+        }
+        internalCluster().fullRestart();
+        ensureYellow("test"); // wait for state recovery
+        state = client().admin().cluster().prepareState().get().getState();
+        assertEquals("true", state.metaData().persistentSettings().get("archived.this.is.unknown"));
+        assertEquals("broken", state.metaData().persistentSettings().get("archived."
+            + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()));
+
+        // delete these settings
+        client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder().putNull("archived.*")).get();
+
+        state = client().admin().cluster().prepareState().get().getState();
+        assertNull(state.metaData().persistentSettings().get("archived.this.is.unknown"));
+        assertNull(state.metaData().persistentSettings().get("archived."
+            + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()));
+        assertHitCount(client().prepareSearch().setQuery(matchAllQuery()).get(), 1L);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
index 422aea70134..4be0cf15afc 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
@@ -28,6 +28,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESAllocationTestCase;
 
 import java.util.HashMap;
@@ -172,12 +173,12 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
                             boolean stateInMemory,
                             boolean expectMetaData) throws Exception {
         MetaData inMemoryMetaData = null;
-        Set<String> oldIndicesList = emptySet();
+        Set<Index> oldIndicesList = emptySet();
         if (stateInMemory) {
             inMemoryMetaData = event.previousState().metaData();
             oldIndicesList = GatewayMetaState.getRelevantIndices(event.previousState(), event.previousState(), oldIndicesList);
         }
-        Set<String> newIndicesList = GatewayMetaState.getRelevantIndices(event.state(),event.previousState(), oldIndicesList);
+        Set<Index> newIndicesList = GatewayMetaState.getRelevantIndices(event.state(),event.previousState(), oldIndicesList);
         // third, get the actual write info
         Iterator<GatewayMetaState.IndexMetaWriteInfo> indices = GatewayMetaState.resolveStatesToBeWritten(oldIndicesList, newIndicesList, inMemoryMetaData, event.state().metaData()).iterator();
 
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
index 118f32a6564..3a3f5f67e77 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
@@ -19,23 +19,28 @@
 
 package org.elasticsearch.gateway;
 
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.NoopDiscovery;
-import org.elasticsearch.test.cluster.NoopClusterService;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
 
 public class GatewayServiceTests extends ESTestCase {
+
     private GatewayService createService(Settings.Builder settings) {
+        ClusterService clusterService = new ClusterService(Settings.EMPTY, null,
+                new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                null, new ClusterName("ClusterServiceTests"));
         return new GatewayService(Settings.builder()
                 .put("http.enabled", "false")
                 .put("discovery.type", "local")
                 .put(settings.build()).build(),
-                null, new NoopClusterService(), null, null, null, null, new NoopDiscovery());
-
+                null, clusterService, null, null, null, null, new NoopDiscovery(), null, null);
     }
 
     public void testDefaultRecoverAfterTime() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java b/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java
index 00c549ef2f1..115e5b68ff0 100644
--- a/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java
@@ -32,7 +32,6 @@ import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -63,7 +62,6 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.startsWith;
 
 @LuceneTestCase.SuppressFileSystems("ExtrasFS") // TODO: fix test to work with ExtrasFS
 public class MetaDataStateFormatTests extends ESTestCase {
@@ -225,94 +223,14 @@ public class MetaDataStateFormatTests extends ESTestCase {
         msg.append(" after: [").append(checksumAfterCorruption).append("]");
         msg.append(" checksum value after corruption: ").append(actualChecksumAfterCorruption).append("]");
         msg.append(" file: ").append(fileToCorrupt.getFileName().toString()).append(" length: ").append(dir.fileLength(fileToCorrupt.getFileName().toString()));
-        logger.debug(msg.toString());
+        logger.debug("{}", msg.toString());
         assumeTrue("Checksum collision - " + msg.toString(),
                 checksumAfterCorruption != checksumBeforeCorruption // collision
                         || actualChecksumAfterCorruption != checksumBeforeCorruption); // checksum corrupted
         }
     }
 
-    // If the latest version doesn't use the legacy format while previous versions do, then fail hard
-    public void testLatestVersionDoesNotUseLegacy() throws IOException {
-        final ToXContent.Params params = ToXContent.EMPTY_PARAMS;
-        MetaDataStateFormat<MetaData> format = MetaStateService.globalStateFormat(randomFrom(XContentType.values()), params);
-        final Path[] dirs = new Path[2];
-        dirs[0] = createTempDir();
-        dirs[1] = createTempDir();
-        for (Path dir : dirs) {
-            Files.createDirectories(dir.resolve(MetaDataStateFormat.STATE_DIR_NAME));
-        }
-        final Path dir1 = randomFrom(dirs);
-        final int v1 = randomInt(10);
-        // write a first state file in the new format
-        format.write(randomMeta(), v1, dir1);
-
-        // write older state files in the old format but with a newer version
-        final int numLegacyFiles = randomIntBetween(1, 5);
-        for (int i = 0; i < numLegacyFiles; ++i) {
-            final Path dir2 = randomFrom(dirs);
-            final int v2 = v1 + 1 + randomInt(10);
-            try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(format.format(), Files.newOutputStream(dir2.resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve(MetaStateService.GLOBAL_STATE_FILE_PREFIX + v2)))) {
-                xcontentBuilder.startObject();
-                MetaData.Builder.toXContent(randomMeta(), xcontentBuilder, params);
-                xcontentBuilder.endObject();
-            }
-        }
-
-        try {
-            format.loadLatestState(logger, dirs);
-            fail("latest version can not be read");
-        } catch (IllegalStateException ex) {
-            assertThat(ex.getMessage(), startsWith("Could not find a state file to recover from among "));
-        }
-        // write the next state file in the new format and ensure it get's a higher ID
-        final MetaData meta = randomMeta();
-        format.write(meta, v1, dirs);
-        final MetaData metaData = format.loadLatestState(logger, dirs);
-        assertEquals(meta.clusterUUID(), metaData.clusterUUID());
-        final Path path = randomFrom(dirs);
-        final Path[] files = FileSystemUtils.files(path.resolve("_state"));
-        assertEquals(1, files.length);
-        assertEquals("global-" + format.findMaxStateId("global-", dirs) + ".st", files[0].getFileName().toString());
-
-    }
-
-    // If both the legacy and the new format are available for the latest version, prefer the new format
-    public void testPrefersNewerFormat() throws IOException {
-        final ToXContent.Params params = ToXContent.EMPTY_PARAMS;
-        MetaDataStateFormat<MetaData> format = MetaStateService.globalStateFormat(randomFrom(XContentType.values()), params);
-        final Path[] dirs = new Path[2];
-        dirs[0] = createTempDir();
-        dirs[1] = createTempDir();
-        for (Path dir : dirs) {
-            Files.createDirectories(dir.resolve(MetaDataStateFormat.STATE_DIR_NAME));
-        }
-        final long v = randomInt(10);
-
-        MetaData meta = randomMeta();
-        String uuid = meta.clusterUUID();
-
-        // write a first state file in the old format
-        final Path dir2 = randomFrom(dirs);
-        MetaData meta2 = randomMeta();
-        assertFalse(meta2.clusterUUID().equals(uuid));
-        try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(format.format(), Files.newOutputStream(dir2.resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve(MetaStateService.GLOBAL_STATE_FILE_PREFIX + v)))) {
-            xcontentBuilder.startObject();
-            MetaData.Builder.toXContent(randomMeta(), xcontentBuilder, params);
-            xcontentBuilder.endObject();
-        }
-
-        // write a second state file in the new format but with the same version
-        format.write(meta, v, dirs);
-
-        MetaData state = format.loadLatestState(logger, dirs);
-        final Path path = randomFrom(dirs);
-        assertTrue(Files.exists(path.resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + (v+1) + ".st")));
-        assertEquals(state.clusterUUID(), uuid);
-    }
-
     public void testLoadState() throws IOException {
-        final ToXContent.Params params = ToXContent.EMPTY_PARAMS;
         final Path[] dirs = new Path[randomIntBetween(1, 5)];
         int numStates = randomIntBetween(1, 5);
         int numLegacy = randomIntBetween(0, numStates);
@@ -321,7 +239,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
             meta.add(randomMeta());
         }
         Set<Path> corruptedFiles = new HashSet<>();
-        MetaDataStateFormat<MetaData> format = MetaStateService.globalStateFormat(randomFrom(XContentType.values()), params);
+        MetaDataStateFormat<MetaData> format = metaDataFormat(randomFrom(XContentType.values()));
         for (int i = 0; i < dirs.length; i++) {
             dirs[i] = createTempDir();
             Files.createDirectories(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME));
@@ -331,9 +249,10 @@ public class MetaDataStateFormatTests extends ESTestCase {
                     Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-"+j);
                     Files.createFile(file); // randomly create 0-byte files -- there is extra logic to skip them
                 } else {
-                    try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(type, Files.newOutputStream(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j)))) {
+                    try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(type,
+                        Files.newOutputStream(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j)))) {
                         xcontentBuilder.startObject();
-                        MetaData.Builder.toXContent(meta.get(j), xcontentBuilder, params);
+                        MetaData.Builder.toXContent(meta.get(j), xcontentBuilder, ToXContent.EMPTY_PARAMS);
                         xcontentBuilder.endObject();
                     }
                 }
@@ -380,7 +299,20 @@ public class MetaDataStateFormatTests extends ESTestCase {
                 assertThat(ExceptionsHelper.unwrap(ex, CorruptStateException.class), notNullValue());
             }
         }
+    }
 
+    private static MetaDataStateFormat<MetaData> metaDataFormat(XContentType format) {
+        return new MetaDataStateFormat<MetaData>(format, MetaData.GLOBAL_STATE_FILE_PREFIX) {
+            @Override
+            public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
+                MetaData.Builder.toXContent(state, builder, ToXContent.EMPTY_PARAMS);
+            }
+
+            @Override
+            public MetaData fromXContent(XContentParser parser) throws IOException {
+                return MetaData.Builder.fromXContent(parser);
+            }
+        };
     }
 
     private MetaData randomMeta() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java
index bada7faa8c8..4999ef5eac5 100644
--- a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.NodeEnvironment;
@@ -68,14 +67,15 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
         index(index, "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
         ensureGreen();
         assertIndexInMetaState(node1, index);
-        assertIndexDirectoryDeleted(node2, index);
+        Index resolveIndex = resolveIndex(index);
+        assertIndexDirectoryDeleted(node2, resolveIndex);
         assertIndexInMetaState(masterNode, index);
 
         logger.debug("relocating index...");
         client().admin().indices().prepareUpdateSettings(index).setSettings(Settings.builder().put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "_name", node2)).get();
         client().admin().cluster().prepareHealth().setWaitForRelocatingShards(0).get();
         ensureGreen();
-        assertIndexDirectoryDeleted(node1, index);
+        assertIndexDirectoryDeleted(node1, resolveIndex);
         assertIndexInMetaState(node2, index);
         assertIndexInMetaState(masterNode, index);
     }
@@ -142,14 +142,16 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
 
         // finally check that meta data is also written of index opened again
         assertAcked(client().admin().indices().prepareOpen(index).get());
+        // make sure index is fully initialized and nothing is changed anymore
+        ensureGreen();
         indicesMetaData = getIndicesMetaDataOnNode(dataNode);
         assertThat(indicesMetaData.get(index).getState(), equalTo(IndexMetaData.State.OPEN));
     }
 
-    protected void assertIndexDirectoryDeleted(final String nodeName, final String indexName) throws Exception {
+    protected void assertIndexDirectoryDeleted(final String nodeName, final Index index) throws Exception {
         assertBusy(() -> {
             logger.info("checking if index directory exists...");
-            assertFalse("Expecting index directory of " + indexName + " to be deleted from node " + nodeName, indexDirectoryExists(nodeName, indexName));
+            assertFalse("Expecting index directory of " + index + " to be deleted from node " + nodeName, indexDirectoryExists(nodeName, index));
         }
         );
     }
@@ -168,9 +170,9 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
     }
 
 
-    private boolean indexDirectoryExists(String nodeName, String indexName) {
+    private boolean indexDirectoryExists(String nodeName, Index index) {
         NodeEnvironment nodeEnv = ((InternalTestCluster) cluster()).getInstance(NodeEnvironment.class, nodeName);
-        for (Path path : nodeEnv.indexPaths(indexName)) {
+        for (Path path : nodeEnv.indexPaths(index)) {
             if (Files.exists(path)) {
                 return true;
             }
diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java b/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java
index 8bcb9c45402..82c38748a48 100644
--- a/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java
@@ -24,6 +24,7 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -40,24 +41,24 @@ public class MetaStateServiceTests extends ESTestCase {
 
     public void testWriteLoadIndex() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(randomSettings(), env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
 
             IndexMetaData index = IndexMetaData.builder("test1").settings(indexSettings).build();
-            metaStateService.writeIndex("test_write", index, null);
-            assertThat(metaStateService.loadIndexState("test1"), equalTo(index));
+            metaStateService.writeIndex("test_write", index);
+            assertThat(metaStateService.loadIndexState(index.getIndex()), equalTo(index));
         }
     }
 
     public void testLoadMissingIndex() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(randomSettings(), env);
-            assertThat(metaStateService.loadIndexState("test1"), nullValue());
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            assertThat(metaStateService.loadIndexState(new Index("test1", "test1UUID")), nullValue());
         }
     }
 
     public void testWriteLoadGlobal() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(randomSettings(), env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
 
             MetaData metaData = MetaData.builder()
                     .persistentSettings(Settings.builder().put("test1", "value1").build())
@@ -69,7 +70,7 @@ public class MetaStateServiceTests extends ESTestCase {
 
     public void testWriteGlobalStateWithIndexAndNoIndexIsLoaded() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(randomSettings(), env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
 
             MetaData metaData = MetaData.builder()
                     .persistentSettings(Settings.builder().put("test1", "value1").build())
@@ -85,7 +86,7 @@ public class MetaStateServiceTests extends ESTestCase {
 
     public void testLoadGlobal() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(randomSettings(), env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
 
             IndexMetaData index = IndexMetaData.builder("test1").settings(indexSettings).build();
             MetaData metaData = MetaData.builder()
@@ -94,7 +95,7 @@ public class MetaStateServiceTests extends ESTestCase {
                     .build();
 
             metaStateService.writeGlobalState("test_write", metaData);
-            metaStateService.writeIndex("test_write", index, null);
+            metaStateService.writeIndex("test_write", index);
 
             MetaData loadedState = metaStateService.loadFullState();
             assertThat(loadedState.persistentSettings(), equalTo(metaData.persistentSettings()));
@@ -102,12 +103,4 @@ public class MetaStateServiceTests extends ESTestCase {
             assertThat(loadedState.index("test1"), equalTo(index));
         }
     }
-
-    private Settings randomSettings() {
-        Settings.Builder builder = Settings.builder();
-        if (randomBoolean()) {
-            builder.put(MetaStateService.FORMAT_SETTING, randomFrom(XContentType.values()).shortName());
-        }
-        return builder.build();
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/gateway/PriorityComparatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PriorityComparatorTests.java
index 6da00d822a2..bb5a6ff748e 100644
--- a/core/src/test/java/org/elasticsearch/gateway/PriorityComparatorTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/PriorityComparatorTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
@@ -37,7 +38,7 @@ import java.util.Map;
 public class PriorityComparatorTests extends ESTestCase {
 
     public void testPreferNewIndices() {
-        RoutingNodes.UnassignedShards shards = new RoutingNodes.UnassignedShards((RoutingNodes) null);
+        RoutingNodes.UnassignedShards shards = new RoutingNodes.UnassignedShards(null);
         List<ShardRouting> shardRoutings = Arrays.asList(TestShardRouting.newShardRouting("oldest", 0, null, null, null,
                 randomBoolean(), ShardRoutingState.UNASSIGNED, new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar")), TestShardRouting.newShardRouting("newest", 0, null, null, null,
                 randomBoolean(), ShardRoutingState.UNASSIGNED, new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar")));
@@ -47,11 +48,11 @@ public class PriorityComparatorTests extends ESTestCase {
         }
         shards.sort(new PriorityComparator() {
             @Override
-            protected Settings getIndexSettings(String index) {
-                if ("oldest".equals(index)) {
+            protected Settings getIndexSettings(Index index) {
+                if ("oldest".equals(index.getName())) {
                     return Settings.builder().put(IndexMetaData.SETTING_CREATION_DATE, 10)
                             .put(IndexMetaData.SETTING_PRIORITY, 1).build();
-                } else if ("newest".equals(index)) {
+                } else if ("newest".equals(index.getName())) {
                     return Settings.builder().put(IndexMetaData.SETTING_CREATION_DATE, 100)
                             .put(IndexMetaData.SETTING_PRIORITY, 1).build();
                 }
@@ -77,11 +78,11 @@ public class PriorityComparatorTests extends ESTestCase {
         }
         shards.sort(new PriorityComparator() {
             @Override
-            protected Settings getIndexSettings(String index) {
-                if ("oldest".equals(index)) {
+            protected Settings getIndexSettings(Index index) {
+                if ("oldest".equals(index.getName())) {
                     return Settings.builder().put(IndexMetaData.SETTING_CREATION_DATE, 10)
                             .put(IndexMetaData.SETTING_PRIORITY, 100).build();
-                } else if ("newest".equals(index)) {
+                } else if ("newest".equals(index.getName())) {
                     return Settings.builder().put(IndexMetaData.SETTING_CREATION_DATE, 100)
                             .put(IndexMetaData.SETTING_PRIORITY, 1).build();
                 }
@@ -118,8 +119,8 @@ public class PriorityComparatorTests extends ESTestCase {
         }
         shards.sort(new PriorityComparator() {
             @Override
-            protected Settings getIndexSettings(String index) {
-                IndexMeta indexMeta = map.get(index);
+            protected Settings getIndexSettings(Index index) {
+                IndexMeta indexMeta = map.get(index.getName());
                 return indexMeta.settings;
             }
         });
diff --git a/core/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java b/core/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java
index 399ef9badab..a1d16bfd884 100644
--- a/core/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java
@@ -82,7 +82,7 @@ public class QuorumGatewayIT extends ESIntegTestCase {
                     assertTrue(awaitBusy(() -> {
                         logger.info("--> running cluster_health (wait for the shards to startup)");
                         ClusterHealthResponse clusterHealth = activeClient.admin().cluster().health(clusterHealthRequest().waitForYellowStatus().waitForNodes("2").waitForActiveShards(test.numPrimaries * 2)).actionGet();
-                        logger.info("--> done cluster_health, status " + clusterHealth.getStatus());
+                        logger.info("--> done cluster_health, status {}", clusterHealth.getStatus());
                         return (!clusterHealth.isTimedOut()) && clusterHealth.getStatus() == ClusterHealthStatus.YELLOW;
                     }, 30, TimeUnit.SECONDS));
                     logger.info("--> one node is closed -- index 1 document into the remaining nodes");
diff --git a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
index 8fd6e303220..dac284ee594 100644
--- a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
@@ -19,11 +19,13 @@
 
 package org.elasticsearch.gateway;
 
+import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.common.settings.Settings;
@@ -37,11 +39,13 @@ import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.InternalTestCluster.RestartCallback;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.store.MockFSDirectoryService;
 import org.elasticsearch.test.store.MockFSIndexStore;
 
 import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.IntStream;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
@@ -88,10 +92,13 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
         assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("appAccountIds", 179)).execute().actionGet(), 2);
         ensureYellow("test"); // wait for primary allocations here otherwise if we have a lot of shards we might have a
         // shard that is still in post recovery when we restart and the ensureYellow() below will timeout
+
+        Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null);
         internalCluster().fullRestart();
 
         logger.info("Running Cluster Health (wait for the shards to startup)");
         ensureYellow();
+        primaryTerms = assertAndCapturePrimaryTerms(primaryTerms);
 
         client().admin().indices().prepareRefresh().execute().actionGet();
         assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("appAccountIds", 179)).execute().actionGet(), 2);
@@ -100,11 +107,37 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
 
         logger.info("Running Cluster Health (wait for the shards to startup)");
         ensureYellow();
+        primaryTerms = assertAndCapturePrimaryTerms(primaryTerms);
 
         client().admin().indices().prepareRefresh().execute().actionGet();
         assertHitCount(client().prepareSearch().setSize(0).setQuery(termQuery("appAccountIds", 179)).execute().actionGet(), 2);
     }
 
+    private Map<String, long[]> assertAndCapturePrimaryTerms(Map<String, long[]> previousTerms) {
+        if (previousTerms == null) {
+            previousTerms = new HashMap<>();
+        }
+        final Map<String, long[]> result = new HashMap<>();
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        for (ObjectCursor<IndexMetaData> cursor : state.metaData().indices().values()) {
+            final IndexMetaData indexMetaData = cursor.value;
+            final String index = indexMetaData.getIndex().getName();
+            final long[] previous = previousTerms.get(index);
+            final long[] current = IntStream.range(0, indexMetaData.getNumberOfShards()).mapToLong(indexMetaData::primaryTerm).toArray();
+            if (previous == null) {
+                result.put(index, current);
+            } else {
+                assertThat("number of terms changed for index [" + index + "]", current.length, equalTo(previous.length));
+                for (int shard = 0; shard < current.length; shard++) {
+                    assertThat("primary term didn't increase for [" + index + "][" + shard + "]", current[shard], greaterThan(previous[shard]));
+                }
+                result.put(index, current);
+            }
+        }
+
+        return result;
+    }
+
     public void testSingleNodeNoFlush() throws Exception {
         internalCluster().startNode();
 
@@ -163,10 +196,14 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
             logger.info("Ensure all primaries have been started");
             ensureYellow();
         }
+
+        Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null);
+
         internalCluster().fullRestart();
 
         logger.info("Running Cluster Health (wait for the shards to startup)");
         ensureYellow();
+        primaryTerms = assertAndCapturePrimaryTerms(primaryTerms);
 
         for (int i = 0; i <= randomInt(10); i++) {
             assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), value1Docs + value2Docs);
@@ -180,6 +217,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
 
         logger.info("Running Cluster Health (wait for the shards to startup)");
         ensureYellow();
+        primaryTerms = assertAndCapturePrimaryTerms(primaryTerms);
 
         for (int i = 0; i <= randomInt(10); i++) {
             assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), value1Docs + value2Docs);
@@ -201,10 +239,13 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
         ensureYellow("test"); // wait for primary allocations here otherwise if we have a lot of shards we might have a
         // shard that is still in post recovery when we restart and the ensureYellow() below will timeout
 
+        Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null);
+
         internalCluster().fullRestart();
 
         logger.info("Running Cluster Health (wait for the shards to startup)");
         ensureYellow();
+        primaryTerms = assertAndCapturePrimaryTerms(primaryTerms);
 
         for (int i = 0; i < 10; i++) {
             assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 2);
@@ -214,6 +255,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
 
         logger.info("Running Cluster Health (wait for the shards to startup)");
         ensureYellow();
+        primaryTerms = assertAndCapturePrimaryTerms(primaryTerms);
 
         for (int i = 0; i < 10; i++) {
             assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 2);
@@ -236,6 +278,8 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
             assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 2);
         }
 
+        Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null);
+
         internalCluster().fullRestart(new RestartCallback() {
             @Override
             public Settings onNodeStopped(String nodeName) throws Exception {
@@ -251,6 +295,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
 
         logger.info("Running Cluster Health (wait for the shards to startup)");
         ensureGreen();
+        primaryTerms = assertAndCapturePrimaryTerms(primaryTerms);
 
         for (int i = 0; i < 10; i++) {
             assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 2);
@@ -276,6 +321,8 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
         String metaDataUuid = client().admin().cluster().prepareState().execute().get().getState().getMetaData().clusterUUID();
         assertThat(metaDataUuid, not(equalTo("_na_")));
 
+        Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null);
+
         logger.info("--> closing first node, and indexing more data to the second node");
         internalCluster().fullRestart(new RestartCallback() {
 
@@ -315,6 +362,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
 
         logger.info("--> running cluster_health (wait for the shards to startup)");
         ensureGreen();
+        primaryTerms = assertAndCapturePrimaryTerms(primaryTerms);
 
         assertThat(client().admin().cluster().prepareState().execute().get().getState().getMetaData().clusterUUID(), equalTo(metaDataUuid));
 
@@ -380,17 +428,21 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
             assertSyncIdsNotNull();
         }
 
-        logger.info("--> disabling allocation while the cluster is shut down", useSyncIds ? "" : " a second time");
+        logger.info("--> disabling allocation while the cluster is shut down{}", useSyncIds ? "" : " a second time");
         // Disable allocations while we are closing nodes
         client().admin().cluster().prepareUpdateSettings()
                 .setTransientSettings(settingsBuilder()
                         .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), EnableAllocationDecider.Allocation.NONE))
                 .get();
+
+        Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null);
+
         logger.info("--> full cluster restart");
         internalCluster().fullRestart();
 
         logger.info("--> waiting for cluster to return to green after {}shutdown", useSyncIds ? "" : "second ");
         ensureGreen();
+        primaryTerms = assertAndCapturePrimaryTerms(primaryTerms);
 
         if (useSyncIds) {
             assertSyncIdsNotNull();
@@ -445,6 +497,8 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
         internalCluster().startNode(settingsBuilder().put(Environment.PATH_DATA_SETTING.getKey(), createTempDir()).build());
 
         ensureGreen();
+        Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null);
+
 
         internalCluster().fullRestart(new RestartCallback() {
 
@@ -455,6 +509,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
         });
 
         ensureYellow();
+        primaryTerms = assertAndCapturePrimaryTerms(primaryTerms);
 
         assertThat(client().admin().indices().prepareExists("test").execute().actionGet().isExists(), equalTo(true));
         assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(), 1);
diff --git a/core/src/test/java/org/elasticsearch/gateway/ReusePeerRecoverySharedTest.java b/core/src/test/java/org/elasticsearch/gateway/ReusePeerRecoverySharedTest.java
index 6f188ef4280..d28e5333225 100644
--- a/core/src/test/java/org/elasticsearch/gateway/ReusePeerRecoverySharedTest.java
+++ b/core/src/test/java/org/elasticsearch/gateway/ReusePeerRecoverySharedTest.java
@@ -100,7 +100,7 @@ public class ReusePeerRecoverySharedTest {
             assertSyncIdsNotNull();
         }
 
-        logger.info("--> disabling allocation while the cluster is shut down", useSyncIds ? "" : " a second time");
+        logger.info("--> disabling allocation while the cluster is shut down{}", useSyncIds ? "" : " a second time");
         // Disable allocations while we are closing nodes
         client().admin().cluster().prepareUpdateSettings().setTransientSettings(
                 settingsBuilder().put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), EnableAllocationDecider.Allocation.NONE))
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java
index 6afe8a0aefc..5d4330ec5c3 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java
@@ -181,7 +181,7 @@ public class NettyHttpServerPipeliningTests extends ESTestCase {
 
         @Override
         public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) {
-            e.getCause().printStackTrace();
+            logger.info("Caught exception", e.getCause());
             e.getChannel().close();
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index 9e0c3776bf1..b7c2c29eb07 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -34,6 +34,7 @@ import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.env.Environment;
@@ -62,8 +63,8 @@ import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.ScriptContextRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.engine.MockEngineFactory;
@@ -194,9 +195,9 @@ public class IndexModuleTests extends ESTestCase {
 
 
     public void testListener() throws IOException {
-        Setting<Boolean> booleanSetting = Setting.boolSetting("foo.bar", false, true, Setting.Scope.INDEX);
+        Setting<Boolean> booleanSetting = Setting.boolSetting("index.foo.bar", false, Property.Dynamic, Property.IndexScope);
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings, booleanSetting), null, new AnalysisRegistry(null, environment));
-        Setting<Boolean> booleanSetting2 = Setting.boolSetting("foo.bar.baz", false, true, Setting.Scope.INDEX);
+        Setting<Boolean> booleanSetting2 = Setting.boolSetting("index.foo.bar.baz", false, Property.Dynamic, Property.IndexScope);
         AtomicBoolean atomicBoolean = new AtomicBoolean(false);
         module.addSettingsUpdateConsumer(booleanSetting, atomicBoolean::set);
 
@@ -332,6 +333,20 @@ public class IndexModuleTests extends ESTestCase {
         indexService.close("simon says", false);
     }
 
+    public void testForceCacheType() throws IOException {
+        Settings indexSettings = Settings.settingsBuilder()
+            .put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), "none")
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
+        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment));
+        module.forceQueryCacheType("custom");
+        module.registerQueryCache("custom", (a, b) -> new CustomQueryCache());
+        IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
+            new IndicesFieldDataCache(settings, listener));
+        assertTrue(indexService.cache().query() instanceof CustomQueryCache);
+        indexService.close("simon says", false);
+    }
+
     class CustomQueryCache implements QueryCache {
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java
index 3a4020e4103..97258b12a3b 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java
@@ -187,12 +187,13 @@ public class IndexServiceTests extends ESSingleNodeTestCase {
                 return ThreadPool.Names.GENERIC;
             }
         };
+
         latch.get().await();
         latch.set(new CountDownLatch(1));
         assertEquals(1, count.get());
-        latch2.get().countDown();
-        latch2.set(new CountDownLatch(1));
-
+        // here we need to swap first before we let it go otherwise threads might be very fast and run that task twice due to
+        // random exception and the schedule interval is 1ms
+        latch2.getAndSet(new CountDownLatch(1)).countDown();
         latch.get().await();
         assertEquals(2, count.get());
         task.close();
diff --git a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
index 677c8358fb0..9d74b1ce429 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
@@ -20,17 +20,21 @@ package org.elasticsearch.index;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
@@ -39,12 +43,13 @@ import java.util.function.Function;
 
 public class IndexSettingsTests extends ESTestCase {
 
-
     public void testRunListener() {
         Version version = VersionUtils.getPreviousVersion();
-        Settings theSettings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build();
+        Settings theSettings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version)
+            .put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build();
         final AtomicInteger integer = new AtomicInteger(0);
-        Setting<Integer> integerSetting = Setting.intSetting("index.test.setting.int", -1, true, Setting.Scope.INDEX);
+        Setting<Integer> integerSetting = Setting.intSetting("index.test.setting.int", -1,
+            Property.Dynamic, Property.IndexScope);
         IndexMetaData metaData = newIndexMeta("index", theSettings);
         IndexSettings settings = newIndexSettings(newIndexMeta("index", theSettings), Settings.EMPTY, integerSetting);
         settings.getScopedSettings().addSettingsUpdateConsumer(integerSetting, integer::set);
@@ -55,7 +60,8 @@ public class IndexSettingsTests extends ESTestCase {
         assertFalse(settings.updateIndexMetaData(metaData));
         assertEquals(metaData.getSettings().getAsMap(), settings.getSettings().getAsMap());
         assertEquals(0, integer.get());
-        assertTrue(settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(theSettings).put("index.test.setting.int", 42).build())));
+        assertTrue(settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(theSettings).put("index.test.setting.int", 42)
+            .build())));
         assertEquals(42, integer.get());
     }
 
@@ -65,21 +71,25 @@ public class IndexSettingsTests extends ESTestCase {
                 .put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build();
         final AtomicInteger integer = new AtomicInteger(0);
         final StringBuilder builder = new StringBuilder();
-        Setting<Integer> integerSetting = Setting.intSetting("index.test.setting.int", -1, true, Setting.Scope.INDEX);
-        Setting<String> notUpdated = new Setting<>("index.not.updated", "", Function.identity(), true, Setting.Scope.INDEX);
+        Setting<Integer> integerSetting = Setting.intSetting("index.test.setting.int", -1,
+            Property.Dynamic, Property.IndexScope);
+        Setting<String> notUpdated = new Setting<>("index.not.updated", "", Function.identity(),
+            Property.Dynamic, Property.IndexScope);
 
         IndexSettings settings = newIndexSettings(newIndexMeta("index", theSettings), Settings.EMPTY, integerSetting, notUpdated);
         settings.getScopedSettings().addSettingsUpdateConsumer(integerSetting, integer::set);
         settings.getScopedSettings().addSettingsUpdateConsumer(notUpdated, builder::append);
         assertEquals(0, integer.get());
         assertEquals("", builder.toString());
-        IndexMetaData newMetaData = newIndexMeta("index", Settings.builder().put(settings.getIndexMetaData().getSettings()).put("index.test.setting.int", 42).build());
+        IndexMetaData newMetaData = newIndexMeta("index", Settings.builder().put(settings.getIndexMetaData().getSettings())
+            .put("index.test.setting.int", 42).build());
         assertTrue(settings.updateIndexMetaData(newMetaData));
         assertSame(settings.getIndexMetaData(), newMetaData);
         assertEquals(42, integer.get());
         assertEquals("", builder.toString());
         integer.set(0);
-        assertTrue(settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(settings.getIndexMetaData().getSettings()).put("index.not.updated", "boom").build())));
+        assertTrue(settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(settings.getIndexMetaData().getSettings())
+            .put("index.not.updated", "boom").build())));
         assertEquals("boom", builder.toString());
         assertEquals("not updated - we preserve the old settings", 0, integer.get());
 
@@ -87,21 +97,25 @@ public class IndexSettingsTests extends ESTestCase {
 
     public void testSettingsConsistency() {
         Version version = VersionUtils.getPreviousVersion();
-        IndexMetaData metaData = newIndexMeta("index", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build());
+        IndexMetaData metaData = newIndexMeta("index", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version)
+            .build());
         IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
         assertEquals(version, settings.getIndexVersionCreated());
         assertEquals("_na_", settings.getUUID());
         try {
-            settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put("index.test.setting.int", 42).build()));
+            settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED,
+                Version.CURRENT).put("index.test.setting.int", 42).build()));
             fail("version has changed");
         } catch (IllegalArgumentException ex) {
             assertTrue(ex.getMessage(), ex.getMessage().startsWith("version mismatch on settings update expected: "));
         }
 
-        metaData = newIndexMeta("index", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build());
+        metaData = newIndexMeta("index", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build());
         settings = new IndexSettings(metaData, Settings.EMPTY);
         try {
-            settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put("index.test.setting.int", 42).build()));
+            settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED,
+                Version.CURRENT).put("index.test.setting.int", 42).build()));
             fail("uuid missing/change");
         } catch (IllegalArgumentException ex) {
             assertEquals("uuid mismatch on settings update expected: 0xdeadbeef but was: _na_", ex.getMessage());
@@ -114,7 +128,8 @@ public class IndexSettingsTests extends ESTestCase {
         if (settings.length > 0) {
             settingSet.addAll(Arrays.asList(settings));
         }
-        return new IndexSettings(metaData, nodeSettings, (idx) -> Regex.simpleMatch(idx, metaData.getIndex().getName()), new IndexScopedSettings(Settings.EMPTY, settingSet));
+        return new IndexSettings(metaData, nodeSettings, (idx) -> Regex.simpleMatch(idx, metaData.getIndex().getName()),
+            new IndexScopedSettings(Settings.EMPTY, settingSet));
     }
 
 
@@ -128,7 +143,7 @@ public class IndexSettingsTests extends ESTestCase {
 
         Settings nodeSettings = Settings.settingsBuilder().put("index.foo.bar", 43).build();
         final AtomicInteger indexValue = new AtomicInteger(0);
-        Setting<Integer> integerSetting = Setting.intSetting("index.foo.bar", -1, true, Setting.Scope.INDEX);
+        Setting<Integer> integerSetting = Setting.intSetting("index.foo.bar", -1, Property.Dynamic, Property.IndexScope);
         IndexSettings settings = newIndexSettings(newIndexMeta("index", theSettings), nodeSettings, integerSetting);
         settings.getScopedSettings().addSettingsUpdateConsumer(integerSetting, indexValue::set);
         assertEquals(numReplicas, settings.getNumberOfReplicas());
@@ -168,7 +183,8 @@ public class IndexSettingsTests extends ESTestCase {
             .build());
         IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
         assertEquals(Translog.Durability.ASYNC, settings.getTranslogDurability());
-        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), "request").build()));
+        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(),
+            "request").build()));
         assertEquals(Translog.Durability.REQUEST, settings.getTranslogDurability());
 
         metaData = newIndexMeta("index", Settings.settingsBuilder()
@@ -185,7 +201,8 @@ public class IndexSettingsTests extends ESTestCase {
             .build());
         IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
         assertFalse(settings.isWarmerEnabled());
-        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_WARMER_ENABLED_SETTING.getKey(), "true").build()));
+        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_WARMER_ENABLED_SETTING.getKey(),
+            "true").build()));
         assertTrue(settings.isWarmerEnabled());
         metaData = newIndexMeta("index", Settings.settingsBuilder()
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
@@ -201,10 +218,13 @@ public class IndexSettingsTests extends ESTestCase {
             .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), refreshInterval)
             .build());
         IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
-        assertEquals(TimeValue.parseTimeValue(refreshInterval, new TimeValue(1, TimeUnit.DAYS), IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()), settings.getRefreshInterval());
+        assertEquals(TimeValue.parseTimeValue(refreshInterval, new TimeValue(1, TimeUnit.DAYS),
+            IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()), settings.getRefreshInterval());
         String newRefreshInterval = getRandomTimeString();
-        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), newRefreshInterval).build()));
-        assertEquals(TimeValue.parseTimeValue(newRefreshInterval, new TimeValue(1, TimeUnit.DAYS), IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()), settings.getRefreshInterval());
+        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(),
+            newRefreshInterval).build()));
+        assertEquals(TimeValue.parseTimeValue(newRefreshInterval, new TimeValue(1, TimeUnit.DAYS),
+            IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()), settings.getRefreshInterval());
     }
 
     private String getRandomTimeString() {
@@ -223,7 +243,8 @@ public class IndexSettingsTests extends ESTestCase {
             .build());
         IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
         assertEquals(15, settings.getMaxResultWindow());
-        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), 42).build()));
+        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(),
+            42).build()));
         assertEquals(42, settings.getMaxResultWindow());
         settings.updateIndexMetaData(newIndexMeta("index", Settings.EMPTY));
         assertEquals(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxResultWindow());
@@ -242,11 +263,15 @@ public class IndexSettingsTests extends ESTestCase {
             .put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(), gcDeleteSetting.getStringRep())
             .build());
         IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
-        assertEquals(TimeValue.parseTimeValue(gcDeleteSetting.getStringRep(), new TimeValue(1, TimeUnit.DAYS), IndexSettings.INDEX_GC_DELETES_SETTING.getKey()).getMillis(), settings.getGcDeletesInMillis());
+        assertEquals(TimeValue.parseTimeValue(gcDeleteSetting.getStringRep(), new TimeValue(1, TimeUnit.DAYS),
+            IndexSettings.INDEX_GC_DELETES_SETTING.getKey()).getMillis(), settings.getGcDeletesInMillis());
         TimeValue newGCDeleteSetting = new TimeValue(Math.abs(randomInt()), TimeUnit.MILLISECONDS);
-        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(), newGCDeleteSetting.getStringRep()).build()));
-        assertEquals(TimeValue.parseTimeValue(newGCDeleteSetting.getStringRep(), new TimeValue(1, TimeUnit.DAYS), IndexSettings.INDEX_GC_DELETES_SETTING.getKey()).getMillis(), settings.getGcDeletesInMillis());
-        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(), randomBoolean() ? -1 : new TimeValue(-1, TimeUnit.MILLISECONDS)).build()));
+        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(),
+            newGCDeleteSetting.getStringRep()).build()));
+        assertEquals(TimeValue.parseTimeValue(newGCDeleteSetting.getStringRep(), new TimeValue(1, TimeUnit.DAYS),
+            IndexSettings.INDEX_GC_DELETES_SETTING.getKey()).getMillis(), settings.getGcDeletesInMillis());
+        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(),
+            randomBoolean() ? -1 : new TimeValue(-1, TimeUnit.MILLISECONDS)).build()));
         assertEquals(-1, settings.getGcDeletesInMillis());
     }
 
@@ -257,7 +282,8 @@ public class IndexSettingsTests extends ESTestCase {
             .build());
         IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
         assertFalse(settings.isTTLPurgeDisabled());
-        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_TTL_DISABLE_PURGE_SETTING.getKey(), "true").build()));
+        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_TTL_DISABLE_PURGE_SETTING.getKey(),
+            "true").build()));
         assertTrue(settings.isTTLPurgeDisabled());
 
         settings.updateIndexMetaData(newIndexMeta("index", Settings.EMPTY));
@@ -272,7 +298,8 @@ public class IndexSettingsTests extends ESTestCase {
 
     public void testTranslogFlushSizeThreshold() {
         ByteSizeValue translogFlushThresholdSize = new ByteSizeValue(Math.abs(randomInt()));
-        ByteSizeValue actualValue = ByteSizeValue.parseBytesSizeValue(translogFlushThresholdSize.toString(), IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey());
+        ByteSizeValue actualValue = ByteSizeValue.parseBytesSizeValue(translogFlushThresholdSize.toString(),
+            IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey());
         IndexMetaData metaData = newIndexMeta("index", Settings.settingsBuilder()
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), translogFlushThresholdSize.toString())
@@ -280,8 +307,33 @@ public class IndexSettingsTests extends ESTestCase {
         IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
         assertEquals(actualValue, settings.getFlushThresholdSize());
         ByteSizeValue newTranslogFlushThresholdSize = new ByteSizeValue(Math.abs(randomInt()));
-        ByteSizeValue actualNewTranslogFlushThresholdSize = ByteSizeValue.parseBytesSizeValue(newTranslogFlushThresholdSize.toString(), IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey());
-        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), newTranslogFlushThresholdSize.toString()).build()));
+        ByteSizeValue actualNewTranslogFlushThresholdSize = ByteSizeValue.parseBytesSizeValue(newTranslogFlushThresholdSize.toString(),
+            IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey());
+        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder()
+            .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), newTranslogFlushThresholdSize.toString()).build()));
         assertEquals(actualNewTranslogFlushThresholdSize, settings.getFlushThresholdSize());
     }
+
+
+    public void testArchiveBrokenIndexSettings() {
+        Settings settings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS.archiveUnknownOrBrokenSettings(Settings.EMPTY);
+        assertSame(settings, Settings.EMPTY);
+        settings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS.archiveUnknownOrBrokenSettings(Settings.builder()
+            .put("index.refresh_interval", "-200").build());
+        assertEquals("-200", settings.get("archived.index.refresh_interval"));
+        assertNull(settings.get("index.refresh_interval"));
+
+        Settings prevSettings = settings; // no double archive
+        settings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS.archiveUnknownOrBrokenSettings(prevSettings);
+        assertSame(prevSettings, settings);
+
+        settings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS.archiveUnknownOrBrokenSettings(Settings.builder()
+            .put("index.version.created", Version.CURRENT.id) // private setting
+            .put("index.unknown", "foo")
+            .put("index.refresh_interval", "2s").build());
+
+        assertEquals("foo", settings.get("archived.index.unknown"));
+        assertEquals(Integer.toString(Version.CURRENT.id), settings.get("index.version.created"));
+        assertEquals("2s", settings.get("index.refresh_interval"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/IndexTests.java b/core/src/test/java/org/elasticsearch/index/IndexTests.java
new file mode 100644
index 00000000000..6ce38c6acba
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/IndexTests.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.apache.lucene.util.TestUtil.randomSimpleString;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.not;
+
+public class IndexTests extends ESTestCase {
+    public void testToString() {
+        assertEquals("[name/uuid]", new Index("name", "uuid").toString());
+        assertEquals("[name]", new Index("name", ClusterState.UNKNOWN_UUID).toString());
+
+        Index random = new Index(randomSimpleString(random(), 1, 100),
+                usually() ? Strings.randomBase64UUID(random()) : ClusterState.UNKNOWN_UUID);
+        assertThat(random.toString(), containsString(random.getName()));
+        if (ClusterState.UNKNOWN_UUID.equals(random.getUUID())) {
+            assertThat(random.toString(), not(containsString(random.getUUID())));
+        } else {
+            assertThat(random.toString(), containsString(random.getUUID()));
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
index a7d127a60c8..aa3da8fc840 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
@@ -156,10 +156,11 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
         assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
         ensureGreen();
         refresh();
-
+        Index index = resolveIndex("foo-copy");
         for (IndicesService service : internalCluster().getDataNodeInstances(IndicesService.class)) {
-            if (service.hasIndex("foo-copy")) {
-                IndexShard shard = service.indexServiceSafe("foo-copy").getShardOrNull(0);
+
+            if (service.hasIndex(index)) {
+                IndexShard shard = service.indexServiceSafe(index).getShardOrNull(0);
                 if (shard.routingEntry().primary()) {
                     assertFalse(shard instanceof ShadowIndexShard);
                 } else {
@@ -201,8 +202,9 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
         IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(IDX).clear().setTranslog(true).get();
         assertEquals(2, indicesStatsResponse.getIndex(IDX).getPrimaries().getTranslog().estimatedNumberOfOperations());
         assertEquals(2, indicesStatsResponse.getIndex(IDX).getTotal().getTranslog().estimatedNumberOfOperations());
+        Index index = resolveIndex(IDX);
         for (IndicesService service : internalCluster().getInstances(IndicesService.class)) {
-            IndexService indexService = service.indexService(IDX);
+            IndexService indexService = service.indexService(index);
             if (indexService != null) {
                 IndexShard shard = indexService.getShard(0);
                 TranslogStats translogStats = shard.translogStats();
diff --git a/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
index e3676366511..9e05122322a 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
@@ -36,24 +36,30 @@ import java.io.IOException;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.startsWith;
 
 public class IndexingSlowLogTests extends ESTestCase {
     public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException {
         BytesReference source = JsonXContent.contentBuilder().startObject().field("foo", "bar").endObject().bytes();
         ParsedDocument pd = new ParsedDocument(new StringField("uid", "test:id", Store.YES), new LegacyIntField("version", 1, Store.YES), "id",
                 "test", null, 0, -1, null, source, null);
-
+        Index index = new Index("foo", "123");
         // Turning off document logging doesn't log source[]
-        SlowLogParsedDocumentPrinter p = new SlowLogParsedDocumentPrinter(pd, 10, true, 0);
+        SlowLogParsedDocumentPrinter p = new SlowLogParsedDocumentPrinter(index, pd, 10, true, 0);
         assertThat(p.toString(), not(containsString("source[")));
 
         // Turning on document logging logs the whole thing
-        p = new SlowLogParsedDocumentPrinter(pd, 10, true, Integer.MAX_VALUE);
+        p = new SlowLogParsedDocumentPrinter(index, pd, 10, true, Integer.MAX_VALUE);
         assertThat(p.toString(), containsString("source[{\"foo\":\"bar\"}]"));
 
         // And you can truncate the source
-        p = new SlowLogParsedDocumentPrinter(pd, 10, true, 3);
+        p = new SlowLogParsedDocumentPrinter(index, pd, 10, true, 3);
         assertThat(p.toString(), containsString("source[{\"f]"));
+
+        // And you can truncate the source
+        p = new SlowLogParsedDocumentPrinter(index, pd, 10, true, 3);
+        assertThat(p.toString(), containsString("source[{\"f]"));
+        assertThat(p.toString(), startsWith("[foo/123] took"));
     }
 
     public void testReformatSetting() {
diff --git a/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
index 7dbff244fcc..e9e8dcfc007 100644
--- a/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
+++ b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.plugins.Plugin;
@@ -43,7 +44,8 @@ public class SettingsListenerIT extends ESIntegTestCase {
 
     public static class SettingsListenerPlugin extends Plugin {
         private final SettingsTestingService service = new SettingsTestingService();
-        private static final Setting<Integer> SETTING = Setting.intSetting("index.test.new.setting", 0, true, Setting.Scope.INDEX);
+        private static final Setting<Integer> SETTING = Setting.intSetting("index.test.new.setting", 0,
+            Property.Dynamic, Property.IndexScope);
         /**
          * The name of the plugin.
          */
@@ -93,7 +95,8 @@ public class SettingsListenerIT extends ESIntegTestCase {
 
     public static class SettingsTestingService {
         public volatile int value;
-        public static Setting<Integer> VALUE = Setting.intSetting("index.test.new.setting", -1, -1, true, Setting.Scope.INDEX);
+        public static Setting<Integer> VALUE = Setting.intSetting("index.test.new.setting", -1, -1,
+            Property.Dynamic, Property.IndexScope);
 
         public void setValue(int value) {
             this.value = value;
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java
index 1fd9655e8de..f665de6216d 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java
@@ -76,7 +76,7 @@ public class AnalysisServiceTests extends ESTestCase {
     }
 
     public void testOverrideDefaultIndexAnalyzer() {
-        Version version = VersionUtils.randomVersionBetween(getRandom(), Version.V_5_0_0, Version.CURRENT);
+        Version version = VersionUtils.randomVersionBetween(getRandom(), Version.V_5_0_0_alpha1, Version.CURRENT);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         try {
             AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
@@ -90,7 +90,7 @@ public class AnalysisServiceTests extends ESTestCase {
     }
 
     public void testBackCompatOverrideDefaultIndexAnalyzer() {
-        Version version = VersionUtils.randomVersionBetween(getRandom(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0));
+        Version version = VersionUtils.randomVersionBetween(getRandom(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1));
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
                 Collections.singletonMap("default_index", analyzerProvider("default_index")),
@@ -112,7 +112,7 @@ public class AnalysisServiceTests extends ESTestCase {
     }
 
     public void testBackCompatOverrideDefaultIndexAndSearchAnalyzer() {
-        Version version = VersionUtils.randomVersionBetween(getRandom(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0));
+        Version version = VersionUtils.randomVersionBetween(getRandom(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1));
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         Map<String, AnalyzerProvider> analyzers = new HashMap<>();
         analyzers.put("default_index", analyzerProvider("default_index"));
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTests.java
index 061e0d9d29f..37943773cef 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTests.java
@@ -21,8 +21,23 @@ package org.elasticsearch.index.analysis;
 
 import org.apache.lucene.analysis.util.CharArraySet;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.BufferedWriter;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.MalformedInputException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.hamcrest.Matchers.is;
 
@@ -42,4 +57,55 @@ public class AnalysisTests extends ESTestCase {
         assertThat(set.contains("bar"), is(true));
         assertThat(set.contains("baz"), is(false));
     }
+
+    public void testParseNonExistingFile() {
+        Path tempDir = createTempDir();
+        Settings nodeSettings = Settings.builder()
+            .put("foo.bar_path", tempDir.resolve("foo.dict"))
+            .put(Environment.PATH_HOME_SETTING.getKey(), tempDir).build();
+        Environment env = new Environment(nodeSettings);
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+            () -> Analysis.getWordList(env, nodeSettings, "foo.bar"));
+        assertEquals("IOException while reading foo.bar_path: " +  tempDir.resolve("foo.dict").toString(), ex.getMessage());
+        assertTrue(ex.getCause().toString(), ex.getCause() instanceof FileNotFoundException
+            || ex.getCause() instanceof NoSuchFileException);
+    }
+
+
+    public void testParseFalseEncodedFile() throws IOException {
+        Path tempDir = createTempDir();
+        Path dict = tempDir.resolve("foo.dict");
+        Settings nodeSettings = Settings.builder()
+            .put("foo.bar_path", dict)
+            .put(Environment.PATH_HOME_SETTING.getKey(), tempDir).build();
+        try (OutputStream writer = Files.newOutputStream(dict)) {
+            writer.write(new byte[]{(byte) 0xff, 0x00, 0x00}); // some invalid UTF-8
+            writer.write('\n');
+        }
+        Environment env = new Environment(nodeSettings);
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+            () -> Analysis.getWordList(env, nodeSettings, "foo.bar"));
+        assertEquals("Unsupported character encoding detected while reading foo.bar_path: " + tempDir.resolve("foo.dict").toString()
+            + " - files must be UTF-8 encoded" , ex.getMessage());
+        assertTrue(ex.getCause().toString(), ex.getCause() instanceof MalformedInputException
+            || ex.getCause() instanceof CharacterCodingException);
+    }
+
+    public void testParseWordList() throws IOException {
+        Path tempDir = createTempDir();
+        Path dict = tempDir.resolve("foo.dict");
+        Settings nodeSettings = Settings.builder()
+            .put("foo.bar_path", dict)
+            .put(Environment.PATH_HOME_SETTING.getKey(), tempDir).build();
+        try (BufferedWriter writer = Files.newBufferedWriter(dict, StandardCharsets.UTF_8)) {
+            writer.write("hello");
+            writer.write('\n');
+            writer.write("world");
+            writer.write('\n');
+        }
+        Environment env = new Environment(nodeSettings);
+        List<String> wordList = Analysis.getWordList(env, nodeSettings, "foo.bar");
+        assertEquals(Arrays.asList("hello", "world"), wordList);
+
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java
index a844f971eac..cf56f41c83a 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
@@ -38,8 +39,10 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFa
 
 @ClusterScope(numDataNodes = 1, scope = Scope.SUITE)
 public class InternalEngineMergeIT extends ESIntegTestCase {
+
+    @TestLogging("_root:DEBUG")
     public void testMergesHappening() throws InterruptedException, IOException, ExecutionException {
-        final int numOfShards = randomIntBetween(1,5);
+        final int numOfShards = randomIntBetween(1, 5);
         // some settings to keep num segments low
         assertAcked(prepareCreate("test").setSettings(Settings.builder()
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numOfShards)
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index e9971a15f8e..ab2041baa4a 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -485,7 +485,7 @@ public class InternalEngineTests extends ESTestCase {
 
             if (flush) {
                 // we should have had just 1 merge, so last generation should be exact
-                assertEquals(gen2 + 1, store.readLastCommittedSegmentsInfo().getLastGeneration());
+                assertEquals(gen2, store.readLastCommittedSegmentsInfo().getLastGeneration());
             }
         }
     }
@@ -843,7 +843,7 @@ public class InternalEngineTests extends ESTestCase {
                         Engine.SyncedFlushResult.SUCCESS);
                 assertEquals(3, engine.segments(false).size());
 
-                engine.forceMerge(false, 1, false, false, false);
+                engine.forceMerge(forceMergeFlushes, 1, false, false, false);
                 if (forceMergeFlushes == false) {
                     engine.refresh("make all segments visible");
                     assertEquals(4, engine.segments(false).size());
@@ -867,7 +867,7 @@ public class InternalEngineTests extends ESTestCase {
                     assertEquals(engine.getLastWriteNanos(), delete.startTime());
                 }
                 assertFalse(engine.tryRenewSyncCommit());
-                engine.flush();
+                engine.flush(false, true); // we might hit a concurrent flush from a finishing merge here - just wait if ongoing...
                 assertNull(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID));
                 assertNull(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID));
             }
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
index 6f8b5a45df0..877dbc83625 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
@@ -81,7 +81,7 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase {
     protected LeafReaderContext readerContext;
     protected DirectoryReader topLevelReader;
     protected IndicesFieldDataCache indicesFieldDataCache;
-    protected abstract FieldDataType getFieldDataType();
+    protected abstract String getFieldDataType();
 
     protected boolean hasDocValues() {
         return false;
@@ -91,7 +91,7 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase {
         return getForField(getFieldDataType(), fieldName, hasDocValues());
     }
 
-    public <IFD extends IndexFieldData<?>> IFD getForField(FieldDataType type, String fieldName) {
+    public <IFD extends IndexFieldData<?>> IFD getForField(String type, String fieldName) {
         return getForField(type, fieldName, hasDocValues());
     }
 
@@ -100,35 +100,35 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase {
         return pluginList(InternalSettingsPlugin.class);
     }
 
-    public <IFD extends IndexFieldData<?>> IFD getForField(FieldDataType type, String fieldName, boolean docValues) {
+    public <IFD extends IndexFieldData<?>> IFD getForField(String type, String fieldName, boolean docValues) {
         final MappedFieldType fieldType;
         final BuilderContext context = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1));
-        if (type.getType().equals("string")) {
-            fieldType = new StringFieldMapper.Builder(fieldName).tokenized(false).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType();
-        } else if (type.getType().equals("float")) {
-            fieldType = new FloatFieldMapper.Builder(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType();
-        } else if (type.getType().equals("double")) {
-            fieldType = new DoubleFieldMapper.Builder(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType();
-        } else if (type.getType().equals("long")) {
-            fieldType = new LongFieldMapper.Builder(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType();
-        } else if (type.getType().equals("int")) {
-            fieldType = new IntegerFieldMapper.Builder(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType();
-        } else if (type.getType().equals("short")) {
-            fieldType = new ShortFieldMapper.Builder(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType();
-        } else if (type.getType().equals("byte")) {
-            fieldType = new ByteFieldMapper.Builder(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType();
-        } else if (type.getType().equals("geo_point")) {
+        if (type.equals("string")) {
+            fieldType = new StringFieldMapper.Builder(fieldName).tokenized(false).fielddata(docValues == false).docValues(docValues).build(context).fieldType();
+        } else if (type.equals("float")) {
+            fieldType = new FloatFieldMapper.Builder(fieldName).docValues(docValues).build(context).fieldType();
+        } else if (type.equals("double")) {
+            fieldType = new DoubleFieldMapper.Builder(fieldName).docValues(docValues).build(context).fieldType();
+        } else if (type.equals("long")) {
+            fieldType = new LongFieldMapper.Builder(fieldName).docValues(docValues).build(context).fieldType();
+        } else if (type.equals("int")) {
+            fieldType = new IntegerFieldMapper.Builder(fieldName).docValues(docValues).build(context).fieldType();
+        } else if (type.equals("short")) {
+            fieldType = new ShortFieldMapper.Builder(fieldName).docValues(docValues).build(context).fieldType();
+        } else if (type.equals("byte")) {
+            fieldType = new ByteFieldMapper.Builder(fieldName).docValues(docValues).build(context).fieldType();
+        } else if (type.equals("geo_point")) {
             if (indexService.getIndexSettings().getIndexVersionCreated().before(Version.V_2_2_0)) {
-                fieldType =  new GeoPointFieldMapperLegacy.Builder(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType();
+                fieldType =  new GeoPointFieldMapperLegacy.Builder(fieldName).docValues(docValues).build(context).fieldType();
             } else {
-                fieldType = new GeoPointFieldMapper.Builder(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType();
+                fieldType = new GeoPointFieldMapper.Builder(fieldName).docValues(docValues).build(context).fieldType();
             }
-        } else if (type.getType().equals("_parent")) {
+        } else if (type.equals("_parent")) {
             fieldType = new ParentFieldMapper.Builder("_type").type(fieldName).build(context).fieldType();
-        } else if (type.getType().equals("binary")) {
-            fieldType = new BinaryFieldMapper.Builder(fieldName).docValues(docValues).fieldDataSettings(type.getSettings()).build(context).fieldType();
+        } else if (type.equals("binary")) {
+            fieldType = new BinaryFieldMapper.Builder(fieldName).docValues(docValues).build(context).fieldType();
         } else {
-            throw new UnsupportedOperationException(type.getType());
+            throw new UnsupportedOperationException(type);
         }
         return ifdService.getForField(fieldType);
     }
@@ -136,8 +136,7 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase {
     @Before
     public void setup() throws Exception {
         Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_3_0); // we need 2.x so that fielddata is allowed on string fields
-        Settings settings = Settings.builder().put("index.fielddata.cache", "none")
-                .put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         indexService = createIndex("test", settings);
         mapperService = indexService.mapperService();
         indicesFieldDataCache = getInstanceFromNode(IndicesService.class).getIndicesFieldDataCache();
@@ -168,7 +167,7 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase {
 
     protected Nested createNested(IndexSearcher searcher, Query parentFilter, Query childFilter) throws IOException {
         BitsetFilterCache s = indexService.cache().bitsetFilterCache();
-        return new Nested(s.getBitSetProducer(parentFilter), searcher.createNormalizedWeight(childFilter, false));
+        return new Nested(s.getBitSetProducer(parentFilter), childFilter);
     }
 
     public void testEmpty() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java
index f11d08f8165..c8d569552db 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java
@@ -38,7 +38,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo;
  */
 public abstract class AbstractGeoFieldDataTestCase extends AbstractFieldDataImplTestCase {
     @Override
-    protected abstract FieldDataType getFieldDataType();
+    protected abstract String getFieldDataType();
 
     protected Field randomGeoPointField(String fieldName, Field.Store store) {
         GeoPoint point = randomPoint(random());
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java
index 15e4790ca9d..a6c4bb15c34 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java
@@ -471,8 +471,7 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI
     public void testGlobalOrdinals() throws Exception {
         fillExtendedMvSet();
         refreshReader();
-        FieldDataType fieldDataType = new FieldDataType("string", Settings.builder().put("global_values", "fixed"));
-        IndexOrdinalsFieldData ifd = getForField(fieldDataType, "value", hasDocValues());
+        IndexOrdinalsFieldData ifd = getForField("string", "value", hasDocValues());
         IndexOrdinalsFieldData globalOrdinals = ifd.loadGlobal(topLevelReader);
         assertThat(topLevelReader.leaves().size(), equalTo(3));
 
@@ -596,8 +595,7 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI
     public void testGlobalOrdinalsGetRemovedOnceIndexReaderCloses() throws Exception {
         fillExtendedMvSet();
         refreshReader();
-        FieldDataType fieldDataType = new FieldDataType("string", Settings.builder().put("global_values", "fixed").put("cache", "node"));
-        IndexOrdinalsFieldData ifd = getForField(fieldDataType, "value", hasDocValues());
+        IndexOrdinalsFieldData ifd = getForField("string", "value", hasDocValues());
         IndexOrdinalsFieldData globalOrdinals = ifd.loadGlobal(topLevelReader);
         assertThat(ifd.loadGlobal(topLevelReader), sameInstance(globalOrdinals));
         // 3 b/c 1 segment level caches and 1 top level cache
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java
index 30669e83409..737d76fec73 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java
@@ -112,7 +112,7 @@ public class BinaryDVFieldDataTests extends AbstractFieldDataTestCase {
     }
 
     @Override
-    protected FieldDataType getFieldDataType() {
-        return new FieldDataType("binary", Settings.builder().put("format", "doc_values"));
+    protected String getFieldDataType() {
+        return "binary";
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/DuelFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/DuelFieldDataTests.java
deleted file mode 100644
index 7ad8653260e..00000000000
--- a/core/src/test/java/org/elasticsearch/index/fielddata/DuelFieldDataTests.java
+++ /dev/null
@@ -1,637 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.fielddata;
-
-import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-import com.carrotsearch.randomizedtesting.generators.RandomStrings;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.SortedSetDocValuesField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.CompositeReaderContext;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.RandomAccessOrds;
-import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.English;
-import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.geo.GeoDistance;
-import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.common.unit.DistanceUnit.Distance;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.ParsedDocument;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.Set;
-
-import static org.hamcrest.Matchers.closeTo;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.lessThan;
-
-public class DuelFieldDataTests extends AbstractFieldDataTestCase {
-    @Override
-    protected FieldDataType getFieldDataType() {
-        return null;
-    }
-
-    public void testDuelAllTypesSingleValue() throws Exception {
-        final String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties")
-                    .startObject("bytes").field("type", "string").field("index", "not_analyzed").endObject()
-                    .startObject("byte").field("type", "byte").endObject()
-                    .startObject("short").field("type", "short").endObject()
-                    .startObject("integer").field("type", "integer").endObject()
-                    .startObject("long").field("type", "long").endObject()
-                    .startObject("float").field("type", "float").endObject()
-                    .startObject("double").field("type", "double").endObject()
-                .endObject().endObject().endObject().string();
-        final DocumentMapper mapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping));
-        Random random = getRandom();
-        int atLeast = scaledRandomIntBetween(200, 1500);
-        for (int i = 0; i < atLeast; i++) {
-            String s = Integer.toString(randomByte());
-
-            XContentBuilder doc = XContentFactory.jsonBuilder().startObject();
-            for (String fieldName : Arrays.asList("bytes", "byte", "short", "integer", "long", "float", "double")) {
-                doc = doc.field(fieldName, s);
-            }
-
-            doc = doc.endObject();
-
-            final ParsedDocument d = mapper.parse("test", "type", Integer.toString(i), doc.bytes());
-
-            writer.addDocument(d.rootDoc());
-
-            if (random.nextInt(10) == 0) {
-                refreshReader();
-            }
-        }
-        LeafReaderContext context = refreshReader();
-        Map<FieldDataType, Type> typeMap = new HashMap<>();
-        typeMap.put(new FieldDataType("string", Settings.builder().put("format", "paged_bytes")), Type.Bytes);
-        typeMap.put(new FieldDataType("byte", Settings.builder().put("format", "doc_values")), Type.Integer);
-        typeMap.put(new FieldDataType("short", Settings.builder().put("format", "doc_values")), Type.Integer);
-        typeMap.put(new FieldDataType("int", Settings.builder().put("format", "doc_values")), Type.Integer);
-        typeMap.put(new FieldDataType("long", Settings.builder().put("format", "doc_values")), Type.Long);
-        typeMap.put(new FieldDataType("double", Settings.builder().put("format", "doc_values")), Type.Double);
-        typeMap.put(new FieldDataType("float", Settings.builder().put("format", "doc_values")), Type.Float);
-        typeMap.put(new FieldDataType("string", Settings.builder().put("format", "doc_values")), Type.Bytes);
-        ArrayList<Entry<FieldDataType, Type>> list = new ArrayList<>(typeMap.entrySet());
-        Preprocessor pre = new ToDoublePreprocessor();
-        while (!list.isEmpty()) {
-            Entry<FieldDataType, Type> left;
-            Entry<FieldDataType, Type> right;
-            if (list.size() > 1) {
-                left = list.remove(random.nextInt(list.size()));
-                right = list.remove(random.nextInt(list.size()));
-            } else {
-                right = left = list.remove(0);
-            }
-
-            ifdService.clear();
-            IndexFieldData<?> leftFieldData = getForField(left.getKey(), left.getValue().name().toLowerCase(Locale.ROOT), true);
-            ifdService.clear();
-            IndexFieldData<?> rightFieldData = getForField(right.getKey(), right.getValue().name().toLowerCase(Locale.ROOT), true);
-            duelFieldDataBytes(random, context, leftFieldData, rightFieldData, pre);
-            duelFieldDataBytes(random, context, rightFieldData, leftFieldData, pre);
-
-            DirectoryReader perSegment = DirectoryReader.open(writer);
-            CompositeReaderContext composite = perSegment.getContext();
-            List<LeafReaderContext> leaves = composite.leaves();
-            for (LeafReaderContext atomicReaderContext : leaves) {
-                duelFieldDataBytes(random, atomicReaderContext, leftFieldData, rightFieldData, pre);
-            }
-        }
-    }
-
-    public void testDuelIntegers() throws Exception {
-        final String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties")
-                    .startObject("byte").field("type", "byte").endObject()
-                    .startObject("short").field("type", "short").endObject()
-                    .startObject("integer").field("type", "integer").endObject()
-                    .startObject("long").field("type", "long").endObject()
-                .endObject().endObject().endObject().string();
-
-        final DocumentMapper mapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping));
-        Random random = getRandom();
-        int atLeast = scaledRandomIntBetween(200, 1500);
-        final int maxNumValues = randomBoolean() ? 1 : randomIntBetween(2, 10);
-        byte[] values = new byte[maxNumValues];
-        for (int i = 0; i < atLeast; i++) {
-            int numValues = randomInt(maxNumValues);
-            // FD loses values if they are duplicated, so we must deduplicate for this test
-            Set<Byte> vals = new HashSet<Byte>();
-            for (int j = 0; j < numValues; ++j) {
-                vals.add(randomByte());
-            }
-
-            numValues = vals.size();
-            int upto = 0;
-            for (Byte bb : vals) {
-                values[upto++] = bb.byteValue();
-            }
-
-            XContentBuilder doc = XContentFactory.jsonBuilder().startObject();
-            for (String fieldName : Arrays.asList("byte", "short", "integer", "long")) {
-                doc = doc.startArray(fieldName);
-                for (int j = 0; j < numValues; ++j) {
-                    doc = doc.value(values[j]);
-                }
-                doc = doc.endArray();
-            }
-            doc = doc.endObject();
-
-            final ParsedDocument d = mapper.parse("test", "type", Integer.toString(i), doc.bytes());
-
-            writer.addDocument(d.rootDoc());
-            if (random.nextInt(10) == 0) {
-                refreshReader();
-            }
-        }
-        LeafReaderContext context = refreshReader();
-        Map<FieldDataType, Type> typeMap = new HashMap<>();
-        typeMap.put(new FieldDataType("byte", Settings.builder().put("format", "doc_values")), Type.Integer);
-        typeMap.put(new FieldDataType("short", Settings.builder().put("format", "doc_values")), Type.Integer);
-        typeMap.put(new FieldDataType("int", Settings.builder().put("format", "doc_values")), Type.Integer);
-        typeMap.put(new FieldDataType("long", Settings.builder().put("format", "doc_values")), Type.Long);
-        ArrayList<Entry<FieldDataType, Type>> list = new ArrayList<>(typeMap.entrySet());
-        while (!list.isEmpty()) {
-            Entry<FieldDataType, Type> left;
-            Entry<FieldDataType, Type> right;
-            if (list.size() > 1) {
-                left = list.remove(random.nextInt(list.size()));
-                right = list.remove(random.nextInt(list.size()));
-            } else {
-                right = left = list.remove(0);
-            }
-            ifdService.clear();
-            IndexNumericFieldData leftFieldData = getForField(left.getKey(), left.getValue().name().toLowerCase(Locale.ROOT), true);
-            ifdService.clear();
-            IndexNumericFieldData rightFieldData = getForField(right.getKey(), right.getValue().name().toLowerCase(Locale.ROOT), true);
-
-            duelFieldDataLong(random, context, leftFieldData, rightFieldData);
-            duelFieldDataLong(random, context, rightFieldData, leftFieldData);
-
-            DirectoryReader perSegment = DirectoryReader.open(writer);
-            CompositeReaderContext composite = perSegment.getContext();
-            List<LeafReaderContext> leaves = composite.leaves();
-            for (LeafReaderContext atomicReaderContext : leaves) {
-                duelFieldDataLong(random, atomicReaderContext, leftFieldData, rightFieldData);
-            }
-        }
-
-    }
-
-    public void testDuelDoubles() throws Exception {
-        final String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties")
-                    .startObject("float").field("type", "float").endObject()
-                    .startObject("double").field("type", "double").endObject()
-                .endObject().endObject().endObject().string();
-
-        final DocumentMapper mapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping));
-        Random random = getRandom();
-        int atLeast = scaledRandomIntBetween(200, 1500);
-        final int maxNumValues = randomBoolean() ? 1 : randomIntBetween(2, 10);
-        float[] values = new float[maxNumValues];
-        for (int i = 0; i < atLeast; i++) {
-            int numValues = randomInt(maxNumValues);
-            float def = randomBoolean() ? randomFloat() : Float.NaN;
-            // FD loses values if they are duplicated, so we must deduplicate for this test
-            Set<Float> vals = new HashSet<Float>();
-            for (int j = 0; j < numValues; ++j) {
-                if (randomBoolean()) {
-                    vals.add(def);
-                } else {
-                    vals.add(randomFloat());
-                }
-            }
-            numValues = vals.size();
-            int upto = 0;
-            for (Float f : vals) {
-                values[upto++] = f.floatValue();
-            }
-
-            XContentBuilder doc = XContentFactory.jsonBuilder().startObject().startArray("float");
-            for (int j = 0; j < numValues; ++j) {
-                doc = doc.value(values[j]);
-            }
-            doc = doc.endArray().startArray("double");
-            for (int j = 0; j < numValues; ++j) {
-                doc = doc.value(values[j]);
-            }
-            doc = doc.endArray().endObject();
-
-            final ParsedDocument d = mapper.parse("test", "type", Integer.toString(i), doc.bytes());
-
-            writer.addDocument(d.rootDoc());
-            if (random.nextInt(10) == 0) {
-                refreshReader();
-            }
-        }
-        LeafReaderContext context = refreshReader();
-        Map<FieldDataType, Type> typeMap = new HashMap<>();
-        typeMap.put(new FieldDataType("double", Settings.builder().put("format", "doc_values")), Type.Double);
-        typeMap.put(new FieldDataType("float", Settings.builder().put("format", "doc_values")), Type.Float);
-        ArrayList<Entry<FieldDataType, Type>> list = new ArrayList<>(typeMap.entrySet());
-        while (!list.isEmpty()) {
-            Entry<FieldDataType, Type> left;
-            Entry<FieldDataType, Type> right;
-            if (list.size() > 1) {
-                left = list.remove(random.nextInt(list.size()));
-                right = list.remove(random.nextInt(list.size()));
-            } else {
-                right = left = list.remove(0);
-            }
-            ifdService.clear();
-            IndexNumericFieldData leftFieldData = getForField(left.getKey(), left.getValue().name().toLowerCase(Locale.ROOT), true);
-
-            ifdService.clear();
-            IndexNumericFieldData rightFieldData = getForField(right.getKey(), right.getValue().name().toLowerCase(Locale.ROOT), true);
-
-            duelFieldDataDouble(random, context, leftFieldData, rightFieldData);
-            duelFieldDataDouble(random, context, rightFieldData, leftFieldData);
-
-            DirectoryReader perSegment = DirectoryReader.open(writer);
-            CompositeReaderContext composite = perSegment.getContext();
-            List<LeafReaderContext> leaves = composite.leaves();
-            for (LeafReaderContext atomicReaderContext : leaves) {
-                duelFieldDataDouble(random, atomicReaderContext, leftFieldData, rightFieldData);
-            }
-        }
-
-    }
-
-    public void testDuelStrings() throws Exception {
-        Random random = getRandom();
-        int atLeast = scaledRandomIntBetween(200, 1500);
-        for (int i = 0; i < atLeast; i++) {
-            Document d = new Document();
-            d.add(new StringField("_id", "" + i, Field.Store.NO));
-            if (random.nextInt(15) != 0) {
-                int[] numbers = getNumbers(random, Integer.MAX_VALUE);
-                for (int j : numbers) {
-                    final String s = English.longToEnglish(j);
-                    d.add(new StringField("bytes", s, Field.Store.NO));
-                    d.add(new SortedSetDocValuesField("bytes", new BytesRef(s)));
-                }
-                if (random.nextInt(10) == 0) {
-                    d.add(new StringField("bytes", "", Field.Store.NO));
-                    d.add(new SortedSetDocValuesField("bytes", new BytesRef()));
-                }
-            }
-            writer.addDocument(d);
-            if (random.nextInt(10) == 0) {
-                refreshReader();
-            }
-        }
-        LeafReaderContext context = refreshReader();
-        Map<FieldDataType, Type> typeMap = new HashMap<>();
-        typeMap.put(new FieldDataType("string", Settings.builder().put("format", "paged_bytes")), Type.Bytes);
-        typeMap.put(new FieldDataType("string", Settings.builder().put("format", "doc_values")), Type.Bytes);
-        // TODO add filters
-        ArrayList<Entry<FieldDataType, Type>> list = new ArrayList<>(typeMap.entrySet());
-        Preprocessor pre = new Preprocessor();
-        while (!list.isEmpty()) {
-            Entry<FieldDataType, Type> left;
-            Entry<FieldDataType, Type> right;
-            if (list.size() > 1) {
-                left = list.remove(random.nextInt(list.size()));
-                right = list.remove(random.nextInt(list.size()));
-            } else {
-                right = left = list.remove(0);
-            }
-            ifdService.clear();
-            IndexFieldData<?> leftFieldData = getForField(left.getKey(), left.getValue().name().toLowerCase(Locale.ROOT));
-
-            ifdService.clear();
-            IndexFieldData<?> rightFieldData = getForField(right.getKey(), right.getValue().name().toLowerCase(Locale.ROOT));
-
-            duelFieldDataBytes(random, context, leftFieldData, rightFieldData, pre);
-            duelFieldDataBytes(random, context, rightFieldData, leftFieldData, pre);
-
-            DirectoryReader perSegment = DirectoryReader.open(writer);
-            CompositeReaderContext composite = perSegment.getContext();
-            List<LeafReaderContext> leaves = composite.leaves();
-            for (LeafReaderContext atomicReaderContext : leaves) {
-                duelFieldDataBytes(random, atomicReaderContext, leftFieldData, rightFieldData, pre);
-            }
-            perSegment.close();
-        }
-
-    }
-
-    public void testDuelGlobalOrdinals() throws Exception {
-        Random random = getRandom();
-        final int numDocs = scaledRandomIntBetween(10, 1000);
-        final int numValues = scaledRandomIntBetween(10, 500);
-        final String[] values = new String[numValues];
-        for (int i = 0; i < numValues; ++i) {
-            values[i] = new String(RandomStrings.randomAsciiOfLength(random, 10));
-        }
-        for (int i = 0; i < numDocs; i++) {
-            Document d = new Document();
-            final int numVals = randomInt(3);
-            for (int j = 0; j < numVals; ++j) {
-                final String value = RandomPicks.randomFrom(random, Arrays.asList(values));
-                d.add(new StringField("string", value, Field.Store.NO));
-                d.add(new SortedSetDocValuesField("bytes", new BytesRef(value)));
-            }
-            writer.addDocument(d);
-            if (randomInt(10) == 0) {
-                refreshReader();
-            }
-        }
-        refreshReader();
-
-        Map<FieldDataType, Type> typeMap = new HashMap<FieldDataType, DuelFieldDataTests.Type>();
-        typeMap.put(new FieldDataType("string", Settings.builder().put("format", "paged_bytes")), Type.Bytes);
-        typeMap.put(new FieldDataType("string", Settings.builder().put("format", "doc_values")), Type.Bytes);
-
-        for (Map.Entry<FieldDataType, Type> entry : typeMap.entrySet()) {
-            ifdService.clear();
-            IndexOrdinalsFieldData fieldData = getForField(entry.getKey(), entry.getValue().name().toLowerCase(Locale.ROOT));
-            RandomAccessOrds left = fieldData.load(readerContext).getOrdinalsValues();
-            fieldData.clear();
-            RandomAccessOrds right = fieldData.loadGlobal(topLevelReader).load(topLevelReader.leaves().get(0)).getOrdinalsValues();
-            assertEquals(left.getValueCount(), right.getValueCount());
-            for (long ord = 0; ord < left.getValueCount(); ++ord) {
-                assertEquals(left.lookupOrd(ord), right.lookupOrd(ord));
-            }
-        }
-    }
-
-    public void testDuelGeoPoints() throws Exception {
-        final String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties")
-                    .startObject("geopoint").field("type", "geo_point").startObject("fielddata").field("format", "doc_values").endObject().endObject()
-                .endObject().endObject().endObject().string();
-
-        final DocumentMapper mapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping));
-
-        Random random = getRandom();
-        int atLeast = scaledRandomIntBetween(200, 1500);
-        int maxValuesPerDoc = randomBoolean() ? 1 : randomIntBetween(2, 10);
-        // to test deduplication
-        double defaultLat = randomDouble() * 180 - 90;
-        double defaultLon = randomDouble() * 360 - 180;
-        for (int i = 0; i < atLeast; i++) {
-            final int numValues = randomInt(maxValuesPerDoc);
-            XContentBuilder doc = XContentFactory.jsonBuilder().startObject().startArray("geopoint");
-            for (int j = 0; j < numValues; ++j) {
-                if (randomBoolean()) {
-                    doc.startObject().field("lat", defaultLat).field("lon", defaultLon).endObject();
-                } else {
-                    doc.startObject().field("lat", randomDouble() * 180 - 90).field("lon", randomDouble() * 360 - 180).endObject();
-                }
-            }
-            doc = doc.endArray().endObject();
-            final ParsedDocument d = mapper.parse("test", "type", Integer.toString(i), doc.bytes());
-
-            writer.addDocument(d.rootDoc());
-            if (random.nextInt(10) == 0) {
-                refreshReader();
-            }
-        }
-        LeafReaderContext context = refreshReader();
-        Map<FieldDataType, Type> typeMap = new HashMap<>();
-        final Distance precision = new Distance(1, randomFrom(DistanceUnit.values()));
-        typeMap.put(new FieldDataType("geo_point", Settings.builder().put("format", "array")), Type.GeoPoint);
-        typeMap.put(new FieldDataType("geo_point", Settings.builder().put("format", "doc_values")), Type.GeoPoint);
-
-        ArrayList<Entry<FieldDataType, Type>> list = new ArrayList<>(typeMap.entrySet());
-        while (!list.isEmpty()) {
-            Entry<FieldDataType, Type> left;
-            Entry<FieldDataType, Type> right;
-            if (list.size() > 1) {
-                left = list.remove(random.nextInt(list.size()));
-                right = list.remove(random.nextInt(list.size()));
-            } else {
-                right = left = list.remove(0);
-            }
-            ifdService.clear();
-            IndexGeoPointFieldData leftFieldData = getForField(left.getKey(), left.getValue().name().toLowerCase(Locale.ROOT));
-
-            ifdService.clear();
-            IndexGeoPointFieldData rightFieldData = getForField(right.getKey(), right.getValue().name().toLowerCase(Locale.ROOT));
-
-            duelFieldDataGeoPoint(random, context, leftFieldData, rightFieldData, precision);
-            duelFieldDataGeoPoint(random, context, rightFieldData, leftFieldData, precision);
-
-            DirectoryReader perSegment = DirectoryReader.open(writer);
-            CompositeReaderContext composite = perSegment.getContext();
-            List<LeafReaderContext> leaves = composite.leaves();
-            for (LeafReaderContext atomicReaderContext : leaves) {
-                duelFieldDataGeoPoint(random, atomicReaderContext, leftFieldData, rightFieldData, precision);
-            }
-            perSegment.close();
-        }
-    }
-
-    @Override
-    public void testEmpty() throws Exception {
-        // No need to test empty usage here
-    }
-
-    private int[] getNumbers(Random random, int margin) {
-        if (random.nextInt(20) == 0) {
-            int[] num = new int[1 + random.nextInt(10)];
-            for (int i = 0; i < num.length; i++) {
-                int v = (random.nextBoolean() ? -1 * random.nextInt(margin) : random.nextInt(margin));
-                num[i] = v;
-            }
-            return num;
-        }
-        return new int[]{(random.nextBoolean() ? -1 * random.nextInt(margin) : random.nextInt(margin))};
-    }
-
-
-    private static void duelFieldDataBytes(Random random, LeafReaderContext context, IndexFieldData<?> left, IndexFieldData<?> right, Preprocessor pre) throws Exception {
-        AtomicFieldData leftData = random.nextBoolean() ? left.load(context) : left.loadDirect(context);
-        AtomicFieldData rightData = random.nextBoolean() ? right.load(context) : right.loadDirect(context);
-
-        int numDocs = context.reader().maxDoc();
-        SortedBinaryDocValues leftBytesValues = leftData.getBytesValues();
-        SortedBinaryDocValues rightBytesValues = rightData.getBytesValues();
-        BytesRefBuilder leftSpare = new BytesRefBuilder();
-        BytesRefBuilder rightSpare = new BytesRefBuilder();
-
-        for (int i = 0; i < numDocs; i++) {
-            leftBytesValues.setDocument(i);
-            rightBytesValues.setDocument(i);
-            int numValues = leftBytesValues.count();
-            assertThat(numValues, equalTo(rightBytesValues.count()));
-            BytesRef previous = null;
-            for (int j = 0; j < numValues; j++) {
-                rightSpare.copyBytes(rightBytesValues.valueAt(j));
-                leftSpare.copyBytes(leftBytesValues.valueAt(j));
-                if (previous != null) {
-                    assertThat(pre.compare(previous, rightSpare.get()), lessThan(0));
-                }
-                previous = BytesRef.deepCopyOf(rightSpare.get());
-                pre.toString(rightSpare.get());
-                pre.toString(leftSpare.get());
-                assertThat(pre.toString(leftSpare.get()), equalTo(pre.toString(rightSpare.get())));
-            }
-        }
-    }
-
-
-    private static void duelFieldDataDouble(Random random, LeafReaderContext context, IndexNumericFieldData left, IndexNumericFieldData right) throws Exception {
-        AtomicNumericFieldData leftData = random.nextBoolean() ? left.load(context) : left.loadDirect(context);
-        AtomicNumericFieldData rightData = random.nextBoolean() ? right.load(context) : right.loadDirect(context);
-
-        int numDocs = context.reader().maxDoc();
-        SortedNumericDoubleValues leftDoubleValues = leftData.getDoubleValues();
-        SortedNumericDoubleValues rightDoubleValues = rightData.getDoubleValues();
-        for (int i = 0; i < numDocs; i++) {
-            leftDoubleValues.setDocument(i);
-            rightDoubleValues.setDocument(i);
-            int numValues = leftDoubleValues.count();
-            assertThat(numValues, equalTo(rightDoubleValues.count()));
-            double previous = 0;
-            for (int j = 0; j < numValues; j++) {
-                double current = rightDoubleValues.valueAt(j);
-                if (Double.isNaN(current)) {
-                    assertTrue(Double.isNaN(leftDoubleValues.valueAt(j)));
-                } else {
-                    assertThat(leftDoubleValues.valueAt(j), closeTo(current, 0.0001));
-                }
-                if (j > 0) {
-                   assertThat(Double.compare(previous,current), lessThan(0));
-                }
-                previous = current;
-            }
-        }
-    }
-
-    private static void duelFieldDataLong(Random random, LeafReaderContext context, IndexNumericFieldData left, IndexNumericFieldData right) throws Exception {
-        AtomicNumericFieldData leftData = random.nextBoolean() ? left.load(context) : left.loadDirect(context);
-        AtomicNumericFieldData rightData = random.nextBoolean() ? right.load(context) : right.loadDirect(context);
-
-        int numDocs = context.reader().maxDoc();
-        SortedNumericDocValues leftLongValues = leftData.getLongValues();
-        SortedNumericDocValues rightLongValues = rightData.getLongValues();
-        for (int i = 0; i < numDocs; i++) {
-            leftLongValues.setDocument(i);
-            rightLongValues.setDocument(i);
-            int numValues = leftLongValues.count();
-            long previous = 0;
-            assertThat(numValues, equalTo(rightLongValues.count()));
-            for (int j = 0; j < numValues; j++) {
-                long current;
-                assertThat(leftLongValues.valueAt(j), equalTo(current = rightLongValues.valueAt(j)));
-                if (j > 0) {
-                    assertThat(previous, lessThan(current));
-                }
-                previous = current;
-            }
-        }
-    }
-
-    private static void duelFieldDataGeoPoint(Random random, LeafReaderContext context, IndexGeoPointFieldData left, IndexGeoPointFieldData right, Distance precision) throws Exception {
-        AtomicGeoPointFieldData leftData = random.nextBoolean() ? left.load(context) : left.loadDirect(context);
-        AtomicGeoPointFieldData rightData = random.nextBoolean() ? right.load(context) : right.loadDirect(context);
-
-        int numDocs = context.reader().maxDoc();
-        MultiGeoPointValues leftValues = leftData.getGeoPointValues();
-        MultiGeoPointValues rightValues = rightData.getGeoPointValues();
-        for (int i = 0; i < numDocs; ++i) {
-            leftValues.setDocument(i);
-            final int numValues = leftValues.count();
-            rightValues.setDocument(i);
-            assertEquals(numValues, rightValues.count());
-            List<GeoPoint> leftPoints = new ArrayList<>();
-            List<GeoPoint> rightPoints = new ArrayList<>();
-            for (int j = 0; j < numValues; ++j) {
-                GeoPoint l = leftValues.valueAt(j);
-                leftPoints.add(new GeoPoint(l.getLat(), l.getLon()));
-                GeoPoint r = rightValues.valueAt(j);
-                rightPoints.add(new GeoPoint(r.getLat(), r.getLon()));
-            }
-            // missing values were treated as 0,0 which are valid geopoints, this now correctly tests for missing values
-            if (leftPoints.isEmpty() == false) {
-                for (GeoPoint l : leftPoints) {
-                    assertTrue("Couldn't find " + l + " among " + rightPoints, contains(l, rightPoints, precision));
-                }
-                for (GeoPoint r : rightPoints) {
-                    assertTrue("Couldn't find " + r + " among " + leftPoints, contains(r, leftPoints, precision));
-                }
-            }
-        }
-    }
-
-    private static boolean contains(GeoPoint point, List<GeoPoint> set, Distance precision) {
-        for (GeoPoint r : set) {
-            final double distance = GeoDistance.PLANE.calculate(point.getLat(), point.getLon(), r.getLat(), r.getLon(), DistanceUnit.METERS);
-            if (new Distance(distance, DistanceUnit.METERS).compareTo(precision) <= 0) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    private static class Preprocessor {
-
-        public String toString(BytesRef ref) {
-            return ref.utf8ToString();
-        }
-
-        public int compare(BytesRef a, BytesRef b) {
-            return a.compareTo(b);
-        }
-    }
-
-    private static class ToDoublePreprocessor extends Preprocessor {
-
-        @Override
-        public String toString(BytesRef ref) {
-            assertTrue(ref.length > 0);
-            return Double.toString(Double.parseDouble(super.toString(ref)));
-        }
-
-        @Override
-        public int compare(BytesRef a, BytesRef b) {
-            Double _a  = Double.parseDouble(super.toString(a));
-            return _a.compareTo(Double.parseDouble(super.toString(b)));
-        }
-    }
-
-
-    private static enum Type {
-        Float, Double, Integer, Long, Bytes, GeoPoint;
-    }
-
-}
-
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java
index 241cbe0e681..01ba314e9c3 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataCacheTests.java
@@ -37,15 +37,12 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
 import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.core.StringFieldMapper;
+import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.FieldMaskingReader;
 
-import java.util.Collections;
-
 import static org.hamcrest.Matchers.equalTo;
 
 public class FieldDataCacheTests extends ESTestCase {
@@ -89,13 +86,14 @@ public class FieldDataCacheTests extends ESTestCase {
     }
 
     private SortedSetDVOrdinalsIndexFieldData createSortedDV(String fieldName, IndexFieldDataCache indexFieldDataCache) {
-        FieldDataType fieldDataType = new StringFieldMapper.StringFieldType().fieldDataType();
-        return new SortedSetDVOrdinalsIndexFieldData(createIndexSettings(), indexFieldDataCache, fieldName, new NoneCircuitBreakerService(), fieldDataType);
+        return new SortedSetDVOrdinalsIndexFieldData(createIndexSettings(), indexFieldDataCache, fieldName, new NoneCircuitBreakerService());
     }
 
     private PagedBytesIndexFieldData createPagedBytes(String fieldName, IndexFieldDataCache indexFieldDataCache) {
-        FieldDataType fieldDataType = new StringFieldMapper.StringFieldType().fieldDataType();
-        return new PagedBytesIndexFieldData(createIndexSettings(), fieldName, fieldDataType, indexFieldDataCache, new NoneCircuitBreakerService());
+        return new PagedBytesIndexFieldData(createIndexSettings(), fieldName, indexFieldDataCache, new NoneCircuitBreakerService(),
+                TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY,
+                TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY,
+                TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE);
     }
 
     private IndexSettings createIndexSettings() {
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataFilterIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataFilterIntegrationIT.java
deleted file mode 100644
index 82b6c995576..00000000000
--- a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataFilterIntegrationIT.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.fielddata;
-
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.search.aggregations.Aggregations;
-import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.hamcrest.Matchers;
-
-import java.io.IOException;
-
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-
-public class FieldDataFilterIntegrationIT extends ESIntegTestCase {
-
-    @Override
-    protected int numberOfReplicas() {
-        return 0;
-    }
-
-    public void testRegexpFilter() throws IOException {
-        CreateIndexRequestBuilder builder = prepareCreate("test");
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties")
-                    .startObject("name")
-                        .field("type", "text")
-                        .startObject("fielddata")
-                            .startObject("filter")
-                                .startObject("regex")
-                                    .field("pattern", "^bac.*")
-                                .endObject()
-                            .endObject()
-                        .endObject()
-                    .endObject()
-                    .startObject("not_filtered")
-                        .field("type", "text")
-                    .endObject()
-                .endObject()
-                .endObject().endObject();
-        assertAcked(builder.addMapping("type", mapping));
-        ensureGreen();
-        int numDocs = scaledRandomIntBetween(5, 50);
-        for (int i = 0; i < numDocs; i++) {
-            client().prepareIndex("test", "type", "" + 0).setSource("name", "bacon bastards", "not_filtered", "bacon bastards").get();
-        }
-        refresh();
-        SearchResponse searchResponse = client().prepareSearch()
-                .setSize(0)
-                .setQuery(matchAllQuery())
-                .addAggregation(terms("name").field("name"))
-                .addAggregation(terms("not_filtered").field("not_filtered")).get();
-        Aggregations aggs = searchResponse.getAggregations();
-        Terms nameAgg = aggs.get("name");
-        assertThat(nameAgg.getBuckets().size(), Matchers.equalTo(1));
-        assertThat(nameAgg.getBuckets().iterator().next().getKeyAsString(), Matchers.equalTo("bacon"));
-
-        Terms notFilteredAgg = aggs.get("not_filtered");
-        assertThat(notFilteredAgg.getBuckets().size(), Matchers.equalTo(2));
-        assertThat(notFilteredAgg.getBuckets().get(0).getKeyAsString(), Matchers.isOneOf("bacon", "bastards"));
-        assertThat(notFilteredAgg.getBuckets().get(1).getKeyAsString(), Matchers.isOneOf("bacon", "bastards"));
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataLoadingIT.java b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataLoadingIT.java
index 8dc46df3405..7b69afa16b5 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataLoadingIT.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataLoadingIT.java
@@ -29,29 +29,13 @@ import static org.hamcrest.Matchers.greaterThan;
 /**
  */
 public class FieldDataLoadingIT extends ESIntegTestCase {
-    public void testEagerFieldDataLoading() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addMapping("type", jsonBuilder().startObject().startObject("type").startObject("properties")
-                        .startObject("name")
-                        .field("type", "text")
-                        .startObject("fielddata").field("loading", "eager").endObject()
-                        .endObject()
-                        .endObject().endObject().endObject()));
-        ensureGreen();
-
-        client().prepareIndex("test", "type", "1").setSource("name", "name").get();
-        client().admin().indices().prepareRefresh("test").get();
-
-        ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get();
-        assertThat(response.getIndicesStats().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
-    }
 
     public void testEagerGlobalOrdinalsFieldDataLoading() throws Exception {
         assertAcked(prepareCreate("test")
                 .addMapping("type", jsonBuilder().startObject().startObject("type").startObject("properties")
                         .startObject("name")
                         .field("type", "text")
-                        .startObject("fielddata").field("loading", "eager_global_ordinals").endObject()
+                        .field("eager_global_ordinals", true)
                         .endObject()
                         .endObject().endObject().endObject()));
         ensureGreen();
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java
index 49cb414208d..14278e1f557 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java
@@ -23,7 +23,10 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.RandomAccessOrds;
-import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.ContentPath;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper.BuilderContext;
+import org.elasticsearch.index.mapper.core.TextFieldMapper;
 
 import java.util.Random;
 
@@ -31,8 +34,7 @@ import static org.hamcrest.Matchers.equalTo;
 
 public class FilterFieldDataTests extends AbstractFieldDataTestCase {
     @Override
-    protected FieldDataType getFieldDataType() {
-        // TODO Auto-generated method stub
+    protected String getFieldDataType() {
         return null;
     }
 
@@ -57,120 +59,56 @@ public class FilterFieldDataTests extends AbstractFieldDataTestCase {
         }
         writer.forceMerge(1, true);
         LeafReaderContext context = refreshReader();
-        String[] formats = new String[] { "paged_bytes"};
+        final BuilderContext builderCtx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1));
 
-        for (String format : formats) {
-            {
-                ifdService.clear();
-                FieldDataType fieldDataType = new FieldDataType("string", Settings.builder().put("format", format)
-                        .put("filter.frequency.min_segment_size", 100).put("filter.frequency.min", 0.0d).put("filter.frequency.max", random.nextBoolean() ? 100 : 0.5d));
-                IndexOrdinalsFieldData fieldData = getForField(fieldDataType, "high_freq");
-                AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
-                RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues();
-                assertThat(2L, equalTo(bytesValues.getValueCount()));
-                assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10"));
-                assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100"));
-            }
-            {
-                ifdService.clear();
-                FieldDataType fieldDataType = new FieldDataType("string", Settings.builder().put("format", format)
-                        .put("filter.frequency.min_segment_size", 100).put("filter.frequency.min",  random.nextBoolean() ? 101 : 101d/200.0d).put("filter.frequency.max", 201));
-                IndexOrdinalsFieldData fieldData = getForField(fieldDataType, "high_freq");
-                AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
-                RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues();
-                assertThat(1L, equalTo(bytesValues.getValueCount()));
-                assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("5"));
-            }
-
-            {
-                ifdService.clear(); // test # docs with value
-                FieldDataType fieldDataType = new FieldDataType("string", Settings.builder().put("format", format)
-                        .put("filter.frequency.min_segment_size", 101).put("filter.frequency.min", random.nextBoolean() ? 101 : 101d/200.0d));
-                IndexOrdinalsFieldData fieldData = getForField(fieldDataType, "med_freq");
-                AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
-                RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues();
-                assertThat(2L, equalTo(bytesValues.getValueCount()));
-                assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10"));
-                assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100"));
-            }
-
-            {
-                ifdService.clear();
-                FieldDataType fieldDataType = new FieldDataType("string", Settings.builder().put("format", format)
-                        .put("filter.frequency.min_segment_size", 101).put("filter.frequency.min", random.nextBoolean() ? 101 : 101d/200.0d));
-                IndexOrdinalsFieldData fieldData = getForField(fieldDataType, "med_freq");
-                AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
-                RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues();
-                assertThat(2L, equalTo(bytesValues.getValueCount()));
-                assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10"));
-                assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100"));
-            }
-
-            {
-                ifdService.clear();
-                FieldDataType fieldDataType = new FieldDataType("string", Settings.builder().put("format", format)
-                        .put("filter.regex.pattern", "\\d{2,3}") // allows 10 & 100
-                        .put("filter.frequency.min_segment_size", 0)
-                        .put("filter.frequency.min", random.nextBoolean() ? 2 : 1d/200.0d) // 100, 10, 5
-                        .put("filter.frequency.max", random.nextBoolean() ? 99 : 99d/200.0d)); // 100
-                IndexOrdinalsFieldData fieldData = getForField(fieldDataType, "high_freq");
-                AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
-                RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues();
-                assertThat(1L, equalTo(bytesValues.getValueCount()));
-                assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("100"));
-            }
+        {
+            ifdService.clear();
+            MappedFieldType ft = new TextFieldMapper.Builder("high_freq")
+                    .fielddataFrequencyFilter(0, random.nextBoolean() ? 100 : 0.5d, 0)
+                    .build(builderCtx).fieldType();
+            IndexOrdinalsFieldData fieldData = ifdService.getForField(ft);
+            AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
+            RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues();
+            assertThat(2L, equalTo(bytesValues.getValueCount()));
+            assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10"));
+            assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100"));
+        }
+        {
+            ifdService.clear();
+            MappedFieldType ft = new TextFieldMapper.Builder("high_freq")
+                    .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d/200.0d, 201, 100)
+                    .build(builderCtx).fieldType();
+            IndexOrdinalsFieldData fieldData = ifdService.getForField(ft);
+            AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
+            RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues();
+            assertThat(1L, equalTo(bytesValues.getValueCount()));
+            assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("5"));
         }
 
-    }
-
-    public void testFilterByRegExp() throws Exception {
-        int hundred  = 0;
-        int ten  = 0;
-        int five  = 0;
-        for (int i = 0; i < 1000; i++) {
-            Document d = new Document();
-            d.add(new StringField("id", "" + i, Field.Store.NO));
-            if (i % 100 == 0) {
-                hundred++;
-                d.add(new StringField("high_freq", "100", Field.Store.NO));
-            }
-            if (i % 10 == 0) {
-                ten++;
-                d.add(new StringField("high_freq", "10", Field.Store.NO));
-            }
-            if (i % 5 == 0) {
-                five++;
-                d.add(new StringField("high_freq", "5", Field.Store.NO));
-
-            }
-            writer.addDocument(d);
+        {
+            ifdService.clear(); // test # docs with value
+            MappedFieldType ft = new TextFieldMapper.Builder("med_freq")
+                    .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d/200.0d, Integer.MAX_VALUE, 101)
+                    .build(builderCtx).fieldType();
+            IndexOrdinalsFieldData fieldData = ifdService.getForField(ft);
+            AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
+            RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues();
+            assertThat(2L, equalTo(bytesValues.getValueCount()));
+            assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10"));
+            assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100"));
         }
-        logger.debug(hundred + " " + ten + " " + five);
-        writer.forceMerge(1, true);
-        LeafReaderContext context = refreshReader();
-        String[] formats = new String[] { "paged_bytes"};
-        for (String format : formats) {
-            {
-                ifdService.clear();
-                FieldDataType fieldDataType = new FieldDataType("string", Settings.builder().put("format", format)
-                        .put("filter.regex.pattern", "\\d"));
-                IndexOrdinalsFieldData fieldData = getForField(fieldDataType, "high_freq");
-                AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
-                RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues();
-                assertThat(1L, equalTo(bytesValues.getValueCount()));
-                assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("5"));
-            }
-            {
-                ifdService.clear();
-                FieldDataType fieldDataType = new FieldDataType("string", Settings.builder().put("format", format)
-                        .put("filter.regex.pattern", "\\d{1,2}"));
-                IndexOrdinalsFieldData fieldData = getForField(fieldDataType, "high_freq");
-                AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
-                RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues();
-                assertThat(2L, equalTo(bytesValues.getValueCount()));
-                assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10"));
-                assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("5"));
-            }
+
+        {
+            ifdService.clear();
+            MappedFieldType ft = new TextFieldMapper.Builder("med_freq")
+                    .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d/200.0d, Integer.MAX_VALUE, 101)
+                    .build(builderCtx).fieldType();
+            IndexOrdinalsFieldData fieldData = ifdService.getForField(ft);
+            AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
+            RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues();
+            assertThat(2L, equalTo(bytesValues.getValueCount()));
+            assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10"));
+            assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100"));
         }
 
     }
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java
index 943af64363e..04ae9bd0840 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java
@@ -35,8 +35,8 @@ public class GeoFieldDataTests extends AbstractGeoFieldDataTestCase {
     private static String FIELD_NAME = "value";
 
     @Override
-    protected FieldDataType getFieldDataType() {
-        return new FieldDataType("geo_point");
+    protected String getFieldDataType() {
+        return "geo_point";
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
index 2d204d1003a..e891f9fab00 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
@@ -46,7 +46,6 @@ import org.elasticsearch.index.mapper.core.IntegerFieldMapper;
 import org.elasticsearch.index.mapper.core.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.core.LongFieldMapper;
 import org.elasticsearch.index.mapper.core.ShortFieldMapper;
-import org.elasticsearch.index.mapper.core.StringFieldMapper;
 import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
@@ -109,7 +108,7 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase {
                 indicesService.getIndicesFieldDataCache(), indicesService.getCircuitBreakerService(), indexService.mapperService());
 
         final BuilderContext ctx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1));
-        final MappedFieldType mapper1 = new TextFieldMapper.Builder("s").build(ctx).fieldType();
+        final MappedFieldType mapper1 = new TextFieldMapper.Builder("s").fielddata(true).build(ctx).fieldType();
         final IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
         Document doc = new Document();
         doc.add(new StringField("s", "thisisastring", Store.NO));
@@ -121,7 +120,7 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase {
         final AtomicInteger onRemovalCalled = new AtomicInteger();
         ifdService.setListener(new IndexFieldDataCache.Listener() {
             @Override
-            public void onCache(ShardId shardId, String fieldName, FieldDataType fieldDataType, Accountable ramUsage) {
+            public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) {
                 if (wrap) {
                     assertEquals(new ShardId("test", "_na_", 1), shardId);
                 } else {
@@ -131,7 +130,7 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase {
             }
 
             @Override
-            public void onRemoval(ShardId shardId, String fieldName, FieldDataType fieldDataType, boolean wasEvicted, long sizeInBytes) {
+            public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) {
                 if (wrap) {
                     assertEquals(new ShardId("test", "_na_", 1), shardId);
                 } else {
@@ -159,12 +158,12 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase {
         try {
             shardPrivateService.setListener(new IndexFieldDataCache.Listener() {
                 @Override
-                public void onCache(ShardId shardId, String fieldName, FieldDataType fieldDataType, Accountable ramUsage) {
+                public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) {
 
                 }
 
                 @Override
-                public void onRemoval(ShardId shardId, String fieldName, FieldDataType fieldDataType, boolean wasEvicted, long sizeInBytes) {
+                public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) {
 
                 }
             });
@@ -206,22 +205,4 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase {
         doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType());
     }
 
-    public void testDisabled() {
-        ThreadPool threadPool = new ThreadPool("random_threadpool_name");
-        StringFieldMapper.StringFieldType ft = new StringFieldMapper.StringFieldType();
-        try {
-            IndicesFieldDataCache cache = new IndicesFieldDataCache(Settings.EMPTY, null);
-            IndexFieldDataService ifds = new IndexFieldDataService(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), cache, null, null);
-            ft.setName("some_str");
-            ft.setFieldDataType(new FieldDataType("string", Settings.builder().put(FieldDataType.FORMAT_KEY, "disabled").build()));
-            try {
-                ifds.getForField(ft);
-                fail();
-            } catch (IllegalStateException e) {
-                assertThat(e.getMessage(), containsString("Field data loading is forbidden on [some_str]"));
-            }
-        } finally {
-            threadPool.shutdown();
-        }
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java
index f1fb694b295..9fdede24d5c 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java
@@ -42,11 +42,6 @@ public class NoOrdinalsStringFieldDataTests extends PagedBytesStringFieldDataTes
                 return in.getFieldName();
             }
 
-            @Override
-            public FieldDataType getFieldDataType() {
-                return in.getFieldDataType();
-            }
-
             @Override
             public AtomicFieldData load(LeafReaderContext context) {
                 return in.load(context);
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/PagedBytesStringFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/PagedBytesStringFieldDataTests.java
index 7a8e879d7ad..25a232f68f9 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/PagedBytesStringFieldDataTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/PagedBytesStringFieldDataTests.java
@@ -19,15 +19,12 @@
 
 package org.elasticsearch.index.fielddata;
 
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
-
 /**
  */
 public class PagedBytesStringFieldDataTests extends AbstractStringFieldDataTestCase {
 
     @Override
-    protected FieldDataType getFieldDataType() {
-        return new FieldDataType("string", Settings.builder().put("format", "paged_bytes").put(OrdinalsBuilder.FORCE_MULTI_ORDINALS, randomBoolean()));
+    protected String getFieldDataType() {
+        return "string";
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java
index 9e1b5d9d167..dc980fcf0a9 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java
@@ -35,11 +35,14 @@ import org.apache.lucene.search.TopFieldDocs;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.MultiValueMode;
 import org.junit.Before;
 
@@ -211,7 +214,8 @@ public class ParentChildFieldDataTests extends AbstractFieldDataTestCase {
 
     public void testThreads() throws Exception {
         final ParentChildIndexFieldData indexFieldData = getForField(childType);
-        final DirectoryReader reader = DirectoryReader.open(writer);
+        final DirectoryReader reader = ElasticsearchDirectoryReader.wrap(
+                DirectoryReader.open(writer), new ShardId(new Index("test", ""), 0));
         final IndexParentChildFieldData global = indexFieldData.loadGlobal(reader);
         final AtomicReference<Exception> error = new AtomicReference<>();
         final int numThreads = scaledRandomIntBetween(3, 8);
@@ -266,7 +270,7 @@ public class ParentChildFieldDataTests extends AbstractFieldDataTestCase {
     }
 
     @Override
-    protected FieldDataType getFieldDataType() {
-        return new FieldDataType("_parent");
+    protected String getFieldDataType() {
+        return "_parent";
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/SortedSetDVStringFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/SortedSetDVStringFieldDataTests.java
index 0b2f174bf05..ceaf13657e3 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/SortedSetDVStringFieldDataTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/SortedSetDVStringFieldDataTests.java
@@ -19,14 +19,11 @@
 
 package org.elasticsearch.index.fielddata;
 
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
-
 public class SortedSetDVStringFieldDataTests extends AbstractStringFieldDataTestCase {
 
     @Override
-    protected FieldDataType getFieldDataType() {
-        return new FieldDataType("string", Settings.builder().put("format", "doc_values").put(OrdinalsBuilder.FORCE_MULTI_ORDINALS, randomBoolean()));
+    protected String getFieldDataType() {
+        return "string";
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/ReplaceMissingTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/ReplaceMissingTests.java
index 63b66f47d1a..a291311c3bc 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/ReplaceMissingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/ReplaceMissingTests.java
@@ -52,7 +52,7 @@ public class ReplaceMissingTests extends ESTestCase {
         iw.close();
 
         DirectoryReader reader = DirectoryReader.open(dir);
-        LeafReader ar = getOnlySegmentReader(reader);
+        LeafReader ar = getOnlyLeafReader(reader);
         SortedDocValues raw = ar.getSortedDocValues("field");
         assertEquals(2, raw.getValueCount());
 
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java
index 655483fa31a..03cc8776e23 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.fielddata.ordinals;
 import org.apache.lucene.index.RandomAccessOrds;
 import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.util.packed.PackedInts;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.test.ESTestCase;
@@ -41,13 +40,8 @@ import static org.hamcrest.Matchers.equalTo;
  */
 public class MultiOrdinalsTests extends ESTestCase {
 
-    protected final Ordinals creationMultiOrdinals(OrdinalsBuilder builder) {
-        return this.creationMultiOrdinals(builder, Settings.builder());
-    }
-
-
-    protected Ordinals creationMultiOrdinals(OrdinalsBuilder builder, Settings.Builder settings) {
-        return builder.build(settings.build());
+    protected Ordinals creationMultiOrdinals(OrdinalsBuilder builder) {
+        return builder.build();
     }
 
     public void testRandomValues() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/SingleOrdinalsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/SingleOrdinalsTests.java
index 3e0d5f6ad2f..b699bb278c9 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/SingleOrdinalsTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/SingleOrdinalsTests.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.fielddata.ordinals;
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.RandomAccessOrds;
 import org.apache.lucene.index.SortedDocValues;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -49,7 +48,7 @@ public class SingleOrdinalsTests extends ESTestCase {
             builder.addDoc(doc);
         }
 
-        Ordinals ords = builder.build(Settings.EMPTY);
+        Ordinals ords = builder.build();
         assertThat(ords, instanceOf(SinglePackedOrdinals.class));
         RandomAccessOrds docs = ords.ordinals();
         final SortedDocValues singleOrds = DocValues.unwrapSingleton(docs);
@@ -68,12 +67,12 @@ public class SingleOrdinalsTests extends ESTestCase {
             builder.addDoc(doc);
         }
 
-        Ordinals ords = builder.build(Settings.EMPTY);
+        Ordinals ords = builder.build();
         assertThat(ords, instanceOf(SinglePackedOrdinals.class));
 
         builder.nextOrdinal();
         builder.addDoc(0);
-        ords = builder.build(Settings.EMPTY);
+        ords = builder.build();
         assertThat(ords, not(instanceOf(SinglePackedOrdinals.class)));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/fieldstats/FieldStatsProviderRefreshTests.java b/core/src/test/java/org/elasticsearch/index/fieldstats/FieldStatsProviderRefreshTests.java
new file mode 100644
index 00000000000..cff2d13ce63
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/fieldstats/FieldStatsProviderRefreshTests.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fieldstats;
+
+import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.indices.IndicesRequestCache;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.equalTo;
+
+public class FieldStatsProviderRefreshTests extends ESSingleNodeTestCase {
+
+    public void testQueryRewriteOnRefresh() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("index").addMapping("type", "s", "type=text")
+                .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true,
+                        IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1,
+                        IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .get());
+
+        // Index some documents
+        indexDocument("1", "d");
+        indexDocument("2", "e");
+        indexDocument("3", "f");
+        refreshIndex();
+
+        // check request cache stats are clean
+        assertRequestCacheStats(0, 0);
+
+        // Search for a range and check that it missed the cache (since its the
+        // first time it has run)
+        final SearchResponse r1 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("s").gte("a").lte("g")).get();
+        assertSearchResponse(r1);
+        assertThat(r1.getHits().getTotalHits(), equalTo(3L));
+        assertRequestCacheStats(0, 1);
+
+        // Search again and check it hits the cache
+        final SearchResponse r2 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("s").gte("a").lte("g")).get();
+        assertSearchResponse(r2);
+        assertThat(r2.getHits().getTotalHits(), equalTo(3L));
+        assertRequestCacheStats(1, 1);
+
+        // Index some more documents in the query range and refresh
+        indexDocument("4", "c");
+        indexDocument("5", "g");
+        refreshIndex();
+
+        // Search again and check the request cache for another miss since request cache should be invalidated by refresh
+        final SearchResponse r3 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("s").gte("a").lte("g")).get();
+        assertSearchResponse(r3);
+        assertThat(r3.getHits().getTotalHits(), equalTo(5L));
+        assertRequestCacheStats(1, 2);
+    }
+
+    private void assertRequestCacheStats(long expectedHits, long expectedMisses) {
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(expectedHits));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(expectedMisses));
+    }
+
+    private void refreshIndex() {
+        RefreshResponse refreshResponse = client().admin().indices().prepareRefresh("index").get();
+        assertThat(refreshResponse.getSuccessfulShards(), equalTo(refreshResponse.getSuccessfulShards()));
+    }
+
+    private void indexDocument(String id, String sValue) {
+        IndexResponse response = client().prepareIndex("index", "type", id).setSource("s", sValue).get();
+        assertThat(response.status(), anyOf(equalTo(RestStatus.OK), equalTo(RestStatus.CREATED)));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/fieldstats/FieldStatsProviderTests.java b/core/src/test/java/org/elasticsearch/index/fieldstats/FieldStatsProviderTests.java
new file mode 100644
index 00000000000..9cad8d3fc8d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/fieldstats/FieldStatsProviderTests.java
@@ -0,0 +1,446 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fieldstats;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.store.BaseDirectoryWrapper;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.engine.Engine.Searcher;
+import org.elasticsearch.index.fieldstats.FieldStatsProvider.Relation;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.similarity.SimilarityService;
+import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.indices.mapper.MapperRegistry;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.chrono.ISOChronology;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import static org.elasticsearch.common.settings.Settings.settingsBuilder;
+import static org.hamcrest.Matchers.equalTo;
+
+public class FieldStatsProviderTests extends ESTestCase {
+
+    private DirectoryReader directoryReader;
+    private Searcher searcher;
+    private FieldStatsProvider fieldStatsProvider;
+    private BaseDirectoryWrapper dir;
+    private AnalysisRegistry analysisRegistry;
+
+    @Before
+    public void setup() throws IOException {
+        Settings nodeSettings = settingsBuilder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
+        IndexSettings settings = IndexSettingsModule.newIndexSettings("_na", nodeSettings);
+        SimilarityService similarityService = new SimilarityService(settings, Collections.emptyMap());
+        analysisRegistry = new AnalysisRegistry(null, new Environment(nodeSettings));
+        AnalysisService analysisService = analysisRegistry.build(settings);
+        IndicesModule indicesModule = new IndicesModule();
+        MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
+        MapperService service = new MapperService(settings, analysisService, similarityService, mapperRegistry, () -> null);
+        putMapping(service);
+        dir = newDirectory();
+        IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
+        indexDocument(service, w, "1", 50L, 50.2f, 50.2, "cherry", new DateTime(2014, 1, 1, 0, 0, 0, ISOChronology.getInstanceUTC()),
+                "10.10.0.10");
+        indexDocument(service, w, "2", 60L, 60.1f, 60.1, "damson", new DateTime(2014, 2, 1, 0, 0, 0, ISOChronology.getInstanceUTC()),
+                "10.10.0.20");
+        indexDocument(service, w, "3", 70L, 70.6f, 70.6, "grape", new DateTime(2014, 3, 1, 0, 0, 0, ISOChronology.getInstanceUTC()),
+                "10.10.0.30");
+        indexDocument(service, w, "4", 80L, 80.2f, 80.2, "kiwi", new DateTime(2014, 4, 1, 0, 0, 0, ISOChronology.getInstanceUTC()),
+                "10.10.0.40");
+        indexDocument(service, w, "5", 90L, 90.4f, 90.4, "lemon", new DateTime(2014, 5, 1, 0, 0, 0, ISOChronology.getInstanceUTC()),
+                "10.10.0.50");
+        indexDocument(service, w, "6", 100L, 100.3f, 100.3, "orange", new DateTime(2014, 6, 1, 0, 0, 0, ISOChronology.getInstanceUTC()),
+                "10.10.0.60");
+        directoryReader = DirectoryReader.open(w, true, true);
+        w.close();
+        ShardId shard = new ShardId("index", "_na_", 0);
+        directoryReader = ElasticsearchDirectoryReader.wrap(directoryReader, shard);
+        IndexSearcher s = new IndexSearcher(directoryReader);
+        searcher = new Engine.Searcher("test", s);
+        fieldStatsProvider = new FieldStatsProvider(searcher, service);
+    }
+
+    @After
+    public void teardown() throws IOException {
+        searcher.close();
+        directoryReader.close();
+        dir.close();
+        analysisRegistry.close();
+    }
+
+    public void testiIsFieldWithinQueryLong() throws IOException {
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 10L, 200L, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 10L, null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", null, 200L, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", null, null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 10L, 100L, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 50L, 200L, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 30L, 80L, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 80L, 200L, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 60L, 80L, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 10L, 100L, true, false, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 50L, 200L, false, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 100L, 200L, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 1L, 50L, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 150L, 200L, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 1L, 8L, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", null, 8L, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 150L, null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 100L, 200L, false, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 1L, 50L, true, false, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+    }
+
+    public void testiIsFieldWithinQueryFloat() throws IOException {
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", 10.8f, 200.5f, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", 10.8f, null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", null, 200.5f, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", null, null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", 10.8f, 100.3f, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", 50.2f, 200.5f, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", 30.5f, 80.1f, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", 80.1f, 200.5f, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", 10.8f, 100.3f, true, false, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", 50.2f, 200.5f, false, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", 100.3f, 200.5f, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", 1.9f, 50.2f, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", 60.9f, 80.1f, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", 150.4f, 200.5f, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", 1.9f, 8.1f, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", null, 8.1f, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", 150.4f, null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", 100.3f, 200.5f, false, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("float_field", 1.9f, 50.2f, true, false, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+    }
+
+    public void testiIsFieldWithinQueryDouble() throws IOException {
+        assertThat(fieldStatsProvider.isFieldWithinQuery("double_field", 10.8, 200.5, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("double_field", 10.8, null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("double_field", null, 200.5, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("double_field", null, null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("double_field", 10.8, 100.3, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("double_field", 50.2, 200.5, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("double_field", 30.5, 80.1, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("double_field", 80.1, 200.5, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("double_field", 60.9, 80.1, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("double_field", 10.8, 100.3, true, false, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("double_field", 50.2, 200.5, false, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("double_field", 100.3, 200.5, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("double_field", 1.9, 50.2, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("double_field", 150.4, 200.5, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("double_field", 1.9, 8.1, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("double_field", null, 8.1, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("double_field", 150.4, null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 100.3, 200.5, false, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("long_field", 1.9, 50.2, true, false, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+    }
+
+    public void testiIsFieldWithinQueryText() throws IOException {
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", new BytesRef("banana"), new BytesRef("zebra"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", new BytesRef("banana"), null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", null, new BytesRef("zebra"), true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", null, null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", new BytesRef("banana"), new BytesRef("orange"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", new BytesRef("cherry"), new BytesRef("zebra"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", new BytesRef("banana"), new BytesRef("grape"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", new BytesRef("grape"), new BytesRef("zebra"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", new BytesRef("lime"), new BytesRef("mango"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", new BytesRef("banana"), new BytesRef("orange"), true, false,
+                DateTimeZone.UTC, null), equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", new BytesRef("cherry"), new BytesRef("zebra"), false, true,
+                DateTimeZone.UTC, null), equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", new BytesRef("orange"), new BytesRef("zebra"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", new BytesRef("apple"), new BytesRef("cherry"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", new BytesRef("peach"), new BytesRef("zebra"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", new BytesRef("apple"), new BytesRef("banana"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", null, new BytesRef("banana"), true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", new BytesRef("peach"), null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", new BytesRef("orange"), new BytesRef("zebra"), false, true,
+                DateTimeZone.UTC, null), equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("text_field", new BytesRef("apple"), new BytesRef("cherry"), true, false,
+                DateTimeZone.UTC, null), equalTo(Relation.DISJOINT));
+    }
+
+    public void testiIsFieldWithinQueryKeyword() throws IOException {
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", new BytesRef("banana"), new BytesRef("zebra"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", new BytesRef("banana"), null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", null, new BytesRef("zebra"), true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", null, null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", new BytesRef("banana"), new BytesRef("orange"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", new BytesRef("cherry"), new BytesRef("zebra"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", new BytesRef("banana"), new BytesRef("grape"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", new BytesRef("grape"), new BytesRef("zebra"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", new BytesRef("lime"), new BytesRef("mango"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", new BytesRef("banana"), new BytesRef("orange"), true, false,
+                DateTimeZone.UTC, null), equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", new BytesRef("cherry"), new BytesRef("zebra"), false, true,
+                DateTimeZone.UTC, null), equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", new BytesRef("orange"), new BytesRef("zebra"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", new BytesRef("apple"), new BytesRef("cherry"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", new BytesRef("peach"), new BytesRef("zebra"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", new BytesRef("apple"), new BytesRef("banana"), true, true,
+                DateTimeZone.UTC, null), equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", null, new BytesRef("banana"), true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", new BytesRef("peach"), null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", new BytesRef("orange"), new BytesRef("zebra"), false, true,
+                DateTimeZone.UTC, null), equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("keyword_field", new BytesRef("apple"), new BytesRef("cherry"), true, false,
+                DateTimeZone.UTC, null), equalTo(Relation.DISJOINT));
+    }
+
+    public void testiIsFieldWithinQueryDate() throws IOException {
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", "2013-01-01", "now", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", "2013-01-01", null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", null, "now", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", null, null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", "2013-01-01", "2014-06-01", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", "2014-01-01", "now", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", "2013-01-01", "2014-03-01", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", "2014-03-01", "now", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", "2014-03-01", "2014-05-01", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", "2013-01-01", "2014-06-01", true, false, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", "2014-01-01", "now", false, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", "2014-06-01", "now", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", "2013-01-01", "2014-01-01", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", "2015-01-01", "now", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", "2013-01-01", "2013-09-01", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", null, "2013-09-01", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", "2015-01-01", null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", "2014-06-01", "now", false, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("date_field", "2013-01-01", "2014-01-01", true, false, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+    }
+
+    public void testiIsFieldWithinQueryIp() throws IOException {
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", "10.10.0.1", "10.20.0.1", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", "10.10.0.1", null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", null, "10.20.0.1", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", null, null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", "10.10.0.1", "10.10.0.60", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", "10.10.0.10", "10.20.0.1", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.WITHIN));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", "10.10.0.1", "10.10.0.40", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", "10.10.0.40", "10.20.0.1", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", "10.10.0.30", "10.10.0.40", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", "10.10.0.1", "10.10.0.60", true, false, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", "10.10.0.10", "10.20.0.1", false, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", "10.10.0.60", "10.20.0.1", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", "10.0.0.1", "10.10.0.10", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.INTERSECTS));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", "10.20.0.10", "10.20.0.1", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", "10.0.0.1", "10.0.0.100", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", null, "10.0.0.100", true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", "10.20.0.10", null, true, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", "10.10.0.60", "10.20.0.1", false, true, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+        assertThat(fieldStatsProvider.isFieldWithinQuery("ip_field", "10.0.0.1", "10.10.0.10", true, false, DateTimeZone.UTC, null),
+                equalTo(Relation.DISJOINT));
+    }
+
+    private void putMapping(MapperService service) throws IOException {
+        XContentBuilder mappingbuilder = JsonXContent.contentBuilder();
+        mappingbuilder.startObject();
+        mappingbuilder.startObject("type");
+        mappingbuilder.startObject("properties");
+        mappingbuilder.startObject("long_field");
+        mappingbuilder.field("type", "long");
+        mappingbuilder.endObject();
+        mappingbuilder.startObject("float_field");
+        mappingbuilder.field("type", "float");
+        mappingbuilder.endObject();
+        mappingbuilder.startObject("double_field");
+        mappingbuilder.field("type", "double");
+        mappingbuilder.endObject();
+        mappingbuilder.startObject("text_field");
+        mappingbuilder.field("type", "text");
+        mappingbuilder.endObject();
+        mappingbuilder.startObject("keyword_field");
+        mappingbuilder.field("type", "keyword");
+        mappingbuilder.endObject();
+        mappingbuilder.startObject("date_field");
+        mappingbuilder.field("type", "date");
+        mappingbuilder.endObject();
+        mappingbuilder.startObject("ip_field");
+        mappingbuilder.field("type", "ip");
+        mappingbuilder.endObject();
+        mappingbuilder.endObject();
+        mappingbuilder.endObject();
+        mappingbuilder.endObject();
+        service.merge("type", new CompressedXContent(mappingbuilder.bytes()), MergeReason.MAPPING_UPDATE, true);
+    }
+
+    private void indexDocument(MapperService service, IndexWriter writer, String id, long longValue, float floatValue, double doubleValue,
+            String stringValue, DateTime dateValue, String ipValue) throws IOException {
+        XContentBuilder docBuilder = JsonXContent.contentBuilder();
+        docBuilder.startObject();
+        docBuilder.field("long_field", longValue);
+        docBuilder.field("float_field", floatValue);
+        docBuilder.field("double_field", doubleValue);
+        docBuilder.field("text_field", stringValue);
+        docBuilder.field("keyword_field", stringValue);
+        docBuilder.field("date_field", dateValue);
+        docBuilder.field("ip_field", ipValue);
+        docBuilder.endObject();
+        DocumentMapper documentMapper = service.documentMapper("type");
+        ParsedDocument doc = documentMapper.parse("index", "type", id, docBuilder.bytes());
+        writer.addDocument(doc.rootDoc());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
index 3206a5e87ae..e4d1e306af3 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
@@ -19,12 +19,20 @@
 
 package org.elasticsearch.index.mapper;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
+import org.elasticsearch.index.mapper.object.ObjectMapper;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
 // TODO: make this a real unit test
 public class DocumentParserTests extends ESSingleNodeTestCase {
 
@@ -61,4 +69,113 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
         assertNotNull(doc.rootDoc().getField("bar"));
         assertNotNull(doc.rootDoc().getField(UidFieldMapper.NAME));
     }
+
+    DocumentMapper createDummyMapping(MapperService mapperService) throws Exception {
+        String mapping = jsonBuilder().startObject().startObject("type").startObject("properties")
+            .startObject("y").field("type", "object").endObject()
+            .startObject("x").startObject("properties")
+            .startObject("subx").field("type", "object").startObject("properties")
+            .startObject("subsubx").field("type", "object")
+            .endObject().endObject().endObject().endObject().endObject().endObject().endObject().endObject().string();
+
+        DocumentMapper defaultMapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping));
+        return defaultMapper;
+    }
+
+    // creates an object mapper, which is about 100x harder than it should be....
+    ObjectMapper createObjectMapper(MapperService mapperService, String name) throws Exception {
+        String[] nameParts = name.split("\\.");
+        ContentPath path = new ContentPath();
+        for (int i = 0; i < nameParts.length - 1; ++i) {
+            path.add(nameParts[i]);
+        }
+        ParseContext context = new ParseContext.InternalParseContext(Settings.EMPTY,
+            mapperService.documentMapperParser(), mapperService.documentMapper("type"), path);
+        Mapper.Builder builder = new ObjectMapper.Builder(nameParts[nameParts.length - 1]).enabled(true);
+        Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path());
+        return (ObjectMapper)builder.build(builderContext);
+    }
+
+    public void testEmptyMappingUpdate() throws Exception {
+        DocumentMapper docMapper = createDummyMapping(createIndex("test").mapperService());
+        assertNull(DocumentParser.createDynamicUpdate(docMapper.mapping(), docMapper, Collections.emptyList()));
+    }
+
+    public void testSingleMappingUpdate() throws Exception {
+        DocumentMapper docMapper = createDummyMapping(createIndex("test").mapperService());
+        List<Mapper> updates = Collections.singletonList(new MockFieldMapper("foo"));
+        Mapping mapping = DocumentParser.createDynamicUpdate(docMapper.mapping(), docMapper, updates);
+        assertNotNull(mapping.root().getMapper("foo"));
+    }
+
+    public void testSubfieldMappingUpdate() throws Exception {
+        DocumentMapper docMapper = createDummyMapping(createIndex("test").mapperService());
+        List<Mapper> updates = Collections.singletonList(new MockFieldMapper("x.foo"));
+        Mapping mapping = DocumentParser.createDynamicUpdate(docMapper.mapping(), docMapper, updates);
+        Mapper xMapper = mapping.root().getMapper("x");
+        assertNotNull(xMapper);
+        assertTrue(xMapper instanceof ObjectMapper);
+        assertNotNull(((ObjectMapper)xMapper).getMapper("foo"));
+        assertNull(((ObjectMapper)xMapper).getMapper("subx"));
+    }
+
+    public void testMultipleSubfieldMappingUpdate() throws Exception {
+        DocumentMapper docMapper = createDummyMapping(createIndex("test").mapperService());
+        List<Mapper> updates = new ArrayList<>();
+        updates.add(new MockFieldMapper("x.foo"));
+        updates.add(new MockFieldMapper("x.bar"));
+        Mapping mapping = DocumentParser.createDynamicUpdate(docMapper.mapping(), docMapper, updates);
+        Mapper xMapper = mapping.root().getMapper("x");
+        assertNotNull(xMapper);
+        assertTrue(xMapper instanceof ObjectMapper);
+        assertNotNull(((ObjectMapper)xMapper).getMapper("foo"));
+        assertNotNull(((ObjectMapper)xMapper).getMapper("bar"));
+        assertNull(((ObjectMapper)xMapper).getMapper("subx"));
+    }
+
+    public void testDeepSubfieldMappingUpdate() throws Exception {
+        DocumentMapper docMapper = createDummyMapping(createIndex("test").mapperService());
+        List<Mapper> updates = Collections.singletonList(new MockFieldMapper("x.subx.foo"));
+        Mapping mapping = DocumentParser.createDynamicUpdate(docMapper.mapping(), docMapper, updates);
+        Mapper xMapper = mapping.root().getMapper("x");
+        assertNotNull(xMapper);
+        assertTrue(xMapper instanceof ObjectMapper);
+        Mapper subxMapper = ((ObjectMapper)xMapper).getMapper("subx");
+        assertTrue(subxMapper instanceof ObjectMapper);
+        assertNotNull(((ObjectMapper)subxMapper).getMapper("foo"));
+        assertNull(((ObjectMapper)subxMapper).getMapper("subsubx"));
+    }
+
+    public void testDeepSubfieldAfterSubfieldMappingUpdate() throws Exception {
+        DocumentMapper docMapper = createDummyMapping(createIndex("test").mapperService());
+        List<Mapper> updates = new ArrayList<>();
+        updates.add(new MockFieldMapper("x.a"));
+        updates.add(new MockFieldMapper("x.subx.b"));
+        Mapping mapping = DocumentParser.createDynamicUpdate(docMapper.mapping(), docMapper, updates);
+        Mapper xMapper = mapping.root().getMapper("x");
+        assertNotNull(xMapper);
+        assertTrue(xMapper instanceof ObjectMapper);
+        assertNotNull(((ObjectMapper)xMapper).getMapper("a"));
+        Mapper subxMapper = ((ObjectMapper)xMapper).getMapper("subx");
+        assertTrue(subxMapper instanceof ObjectMapper);
+        assertNotNull(((ObjectMapper)subxMapper).getMapper("b"));
+    }
+
+    public void testObjectMappingUpdate() throws Exception {
+        MapperService mapperService = createIndex("test").mapperService();
+        DocumentMapper docMapper = createDummyMapping(mapperService);
+        List<Mapper> updates = new ArrayList<>();
+        updates.add(createObjectMapper(mapperService, "foo"));
+        updates.add(createObjectMapper(mapperService, "foo.bar"));
+        updates.add(new MockFieldMapper("foo.bar.baz"));
+        updates.add(new MockFieldMapper("foo.field"));
+        Mapping mapping = DocumentParser.createDynamicUpdate(docMapper.mapping(), docMapper, updates);
+        Mapper fooMapper = mapping.root().getMapper("foo");
+        assertNotNull(fooMapper);
+        assertTrue(fooMapper instanceof ObjectMapper);
+        assertNotNull(((ObjectMapper)fooMapper).getMapper("field"));
+        Mapper barMapper = ((ObjectMapper)fooMapper).getMapper("bar");
+        assertTrue(barMapper instanceof ObjectMapper);
+        assertNotNull(((ObjectMapper)barMapper).getMapper("baz"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
index 748dd0a0a1a..51e88e50edd 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
@@ -28,27 +28,30 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.local.LocalTransport;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.test.cluster.TestClusterService;
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
-import static org.hamcrest.CoreMatchers.instanceOf;
 
 import java.util.Collections;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.hamcrest.CoreMatchers.instanceOf;
+
 public class DynamicMappingDisabledTests extends ESSingleNodeTestCase {
 
     private static ThreadPool THREAD_POOL;
-    private TestClusterService clusterService;
+    private ClusterService clusterService;
     private LocalTransport transport;
     private TransportService transportService;
     private IndicesService indicesService;
@@ -67,9 +70,9 @@ public class DynamicMappingDisabledTests extends ESSingleNodeTestCase {
     public void setUp() throws Exception {
         super.setUp();
         settings = Settings.builder()
-            .put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), false)
-            .build();
-        clusterService = new TestClusterService(THREAD_POOL);
+                .put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), false)
+                .build();
+        clusterService = createClusterService(THREAD_POOL);
         transport = new LocalTransport(settings, THREAD_POOL, Version.CURRENT, new NamedWriteableRegistry());
         transportService = new TransportService(transport, THREAD_POOL);
         indicesService = getInstanceFromNode(IndicesService.class);
@@ -79,6 +82,14 @@ public class DynamicMappingDisabledTests extends ESSingleNodeTestCase {
         autoCreateIndex = new AutoCreateIndex(settings, indexNameExpressionResolver);
     }
 
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        clusterService.close();
+        transportService.close();
+    }
+
+
     @AfterClass
     public static void destroyThreadPool() {
         ThreadPool.terminate(THREAD_POOL, 30, TimeUnit.SECONDS);
@@ -88,8 +99,8 @@ public class DynamicMappingDisabledTests extends ESSingleNodeTestCase {
 
     public void testDynamicDisabled() {
         TransportIndexAction action = new TransportIndexAction(settings, transportService, clusterService,
-            indicesService, THREAD_POOL, shardStateAction, null, null, actionFilters, indexNameExpressionResolver,
-            autoCreateIndex);
+                indicesService, THREAD_POOL, shardStateAction, null, null, actionFilters, indexNameExpressionResolver,
+                autoCreateIndex);
 
         IndexRequest request = new IndexRequest("index", "type", "1");
         request.source("foo", 3);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
index 97c030e9eec..e3a81d2a06e 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
@@ -42,6 +42,7 @@ import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import java.io.IOException;
+import java.util.List;
 
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -211,7 +212,9 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
         ctx.reset(XContentHelper.createParser(source.source()), new ParseContext.Document(), source);
         assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken());
         ctx.parser().nextToken();
-        return DocumentParser.parseObject(ctx, mapper.root(), true);
+        DocumentParser.parseObjectOrNested(ctx, mapper.root(), true);
+        Mapping mapping = DocumentParser.createDynamicUpdate(mapper.mapping(), mapper, ctx.getDynamicMappers());
+        return mapping == null ? null : mapping.root();
     }
 
     public void testDynamicMappingsNotNeeded() throws Exception {
@@ -509,7 +512,13 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
     }
 
     public void testDefaultFloatingPointMappings() throws IOException {
-        DocumentMapper mapper = createIndex("test").mapperService().documentMapperWithAutoCreate("type").getDocumentMapper();
+        MapperService mapperService = createIndex("test").mapperService();
+        String mapping = jsonBuilder().startObject()
+                .startObject("type")
+                    .field("numeric_detection", true)
+                .endObject().endObject().string();
+        mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE, false);
+        DocumentMapper mapper = mapperService.documentMapper("type");
         doTestDefaultFloatingPointMappings(mapper, XContentFactory.jsonBuilder());
         doTestDefaultFloatingPointMappings(mapper, XContentFactory.yamlBuilder());
         doTestDefaultFloatingPointMappings(mapper, XContentFactory.smileBuilder());
@@ -521,6 +530,7 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
                 .field("foo", 3.2f) // float
                 .field("bar", 3.2d) // double
                 .field("baz", (double) 3.2f) // double that can be accurately represented as a float
+                .field("quux", "3.2") // float detected through numeric detection
                 .endObject().bytes();
         ParsedDocument parsedDocument = mapper.parse("index", "type", "id", source);
         Mapping update = parsedDocument.dynamicMappingsUpdate();
@@ -528,5 +538,6 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
         assertThat(update.root().getMapper("foo"), instanceOf(FloatFieldMapper.class));
         assertThat(update.root().getMapper("bar"), instanceOf(FloatFieldMapper.class));
         assertThat(update.root().getMapper("baz"), instanceOf(FloatFieldMapper.class));
+        assertThat(update.root().getMapper("quux"), instanceOf(FloatFieldMapper.class));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java
new file mode 100644
index 00000000000..31b65e84aa0
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.mapper.object.DynamicTemplate;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class DynamicTemplateTests extends ESTestCase {
+
+    public void testParseUnknownParam() throws Exception {
+        Map<String, Object> templateDef = new HashMap<>();
+        templateDef.put("match_mapping_type", "string");
+        templateDef.put("mapping", Collections.singletonMap("store", true));
+        templateDef.put("random_param", "random_value");
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha1));
+        assertEquals("Illegal dynamic template parameter: [random_param]", e.getMessage());
+
+        // but no issues on 2.x for bw compat
+        DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.V_2_3_0);
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        template.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\"match_mapping_type\":\"string\",\"mapping\":{\"store\":true}}", builder.string());
+    }
+
+    public void testSerialization() throws Exception {
+        // type-based template
+        Map<String, Object> templateDef = new HashMap<>();
+        templateDef.put("match_mapping_type", "string");
+        templateDef.put("mapping", Collections.singletonMap("store", true));
+        DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha1);
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        template.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\"match_mapping_type\":\"string\",\"mapping\":{\"store\":true}}", builder.string());
+
+        // name-based template
+        templateDef = new HashMap<>();
+        templateDef.put("match", "*name");
+        templateDef.put("unmatch", "first_name");
+        templateDef.put("mapping", Collections.singletonMap("store", true));
+        template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha1);
+        builder = JsonXContent.contentBuilder();
+        template.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\"match\":\"*name\",\"unmatch\":\"first_name\",\"mapping\":{\"store\":true}}", builder.string());
+
+        // path-based template
+        templateDef = new HashMap<>();
+        templateDef.put("path_match", "*name");
+        templateDef.put("path_unmatch", "first_name");
+        templateDef.put("mapping", Collections.singletonMap("store", true));
+        template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha1);
+        builder = JsonXContent.contentBuilder();
+        template.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\"path_match\":\"*name\",\"path_unmatch\":\"first_name\",\"mapping\":{\"store\":true}}",
+                builder.string());
+
+        // regex matching
+        templateDef = new HashMap<>();
+        templateDef.put("match", "^a$");
+        templateDef.put("match_pattern", "regex");
+        templateDef.put("mapping", Collections.singletonMap("store", true));
+        template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha1);
+        builder = JsonXContent.contentBuilder();
+        template.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\"match\":\"^a$\",\"match_pattern\":\"regex\",\"mapping\":{\"store\":true}}", builder.string());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java
index c5dbd653bfe..cb9a64d357c 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java
@@ -19,12 +19,8 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -59,7 +55,7 @@ public class FieldTypeLookupTests extends ESTestCase {
 
     public void testAddNewField() {
         FieldTypeLookup lookup = new FieldTypeLookup();
-        FakeFieldMapper f = new FakeFieldMapper("foo");
+        MockFieldMapper f = new MockFieldMapper("foo");
         FieldTypeLookup lookup2 = lookup.copyAndAddAll("type", newList(f), randomBoolean());
         assertNull(lookup.get("foo"));
         assertNull(lookup.get("bar"));
@@ -73,8 +69,8 @@ public class FieldTypeLookupTests extends ESTestCase {
     }
 
     public void testAddExistingField() {
-        FakeFieldMapper f = new FakeFieldMapper("foo");
-        FakeFieldMapper f2 = new FakeFieldMapper("foo");
+        MockFieldMapper f = new MockFieldMapper("foo");
+        MockFieldMapper f2 = new MockFieldMapper("foo");
         FieldTypeLookup lookup = new FieldTypeLookup();
         lookup = lookup.copyAndAddAll("type1", newList(f), randomBoolean());
         FieldTypeLookup lookup2 = lookup.copyAndAddAll("type2", newList(f2), randomBoolean());
@@ -84,8 +80,8 @@ public class FieldTypeLookupTests extends ESTestCase {
     }
 
     public void testAddExistingIndexName() {
-        FakeFieldMapper f = new FakeFieldMapper("foo");
-        FakeFieldMapper f2 = new FakeFieldMapper("bar");
+        MockFieldMapper f = new MockFieldMapper("foo");
+        MockFieldMapper f2 = new MockFieldMapper("bar");
         FieldTypeLookup lookup = new FieldTypeLookup();
         lookup = lookup.copyAndAddAll("type1", newList(f), randomBoolean());
         FieldTypeLookup lookup2 = lookup.copyAndAddAll("type2", newList(f2), randomBoolean());
@@ -96,8 +92,8 @@ public class FieldTypeLookupTests extends ESTestCase {
     }
 
     public void testAddExistingFullName() {
-        FakeFieldMapper f = new FakeFieldMapper("foo");
-        FakeFieldMapper f2 = new FakeFieldMapper("foo");
+        MockFieldMapper f = new MockFieldMapper("foo");
+        MockFieldMapper f2 = new MockFieldMapper("foo");
         FieldTypeLookup lookup = new FieldTypeLookup();
         try {
             lookup.copyAndAddAll("type2", newList(f2), randomBoolean());
@@ -107,12 +103,13 @@ public class FieldTypeLookupTests extends ESTestCase {
     }
 
     public void testCheckCompatibilityMismatchedTypes() {
-        FieldMapper f1 = new FakeFieldMapper("foo");
+        FieldMapper f1 = new MockFieldMapper("foo");
         FieldTypeLookup lookup = new FieldTypeLookup();
         lookup = lookup.copyAndAddAll("type", newList(f1), randomBoolean());
 
-        MappedFieldType ft2 = FakeFieldMapper.makeOtherFieldType("foo");
-        FieldMapper f2 = new FakeFieldMapper("foo", ft2);
+        OtherFakeFieldType ft2 = new OtherFakeFieldType();
+        ft2.setName("foo");
+        FieldMapper f2 = new MockFieldMapper("foo", ft2);
         try {
             lookup.copyAndAddAll("type2", newList(f2), false);
             fail("expected type mismatch");
@@ -129,13 +126,14 @@ public class FieldTypeLookupTests extends ESTestCase {
     }
 
     public void testCheckCompatibilityConflict() {
-        FieldMapper f1 = new FakeFieldMapper("foo");
+        FieldMapper f1 = new MockFieldMapper("foo");
         FieldTypeLookup lookup = new FieldTypeLookup();
         lookup = lookup.copyAndAddAll("type", newList(f1), randomBoolean());
 
-        MappedFieldType ft2 = FakeFieldMapper.makeFieldType("foo");
+        MappedFieldType ft2 = new MockFieldMapper.FakeFieldType();
+        ft2.setName("foo");
         ft2.setBoost(2.0f);
-        FieldMapper f2 = new FakeFieldMapper("foo", ft2);
+        FieldMapper f2 = new MockFieldMapper("foo", ft2);
         try {
             // different type
             lookup.copyAndAddAll("type2", newList(f2), false);
@@ -146,9 +144,10 @@ public class FieldTypeLookupTests extends ESTestCase {
         lookup.copyAndAddAll("type", newList(f2), false); // boost is updateable, so ok since we are implicitly updating all types
         lookup.copyAndAddAll("type2", newList(f2), true); // boost is updateable, so ok if forcing
         // now with a non changeable setting
-        MappedFieldType ft3 = FakeFieldMapper.makeFieldType("foo");
+        MappedFieldType ft3 = new MockFieldMapper.FakeFieldType();
+        ft3.setName("foo");
         ft3.setStored(true);
-        FieldMapper f3 = new FakeFieldMapper("foo", ft3);
+        FieldMapper f3 = new MockFieldMapper("foo", ft3);
         try {
             lookup.copyAndAddAll("type2", newList(f3), false);
             fail("expected conflict");
@@ -165,8 +164,8 @@ public class FieldTypeLookupTests extends ESTestCase {
     }
 
     public void testSimpleMatchFullNames() {
-        FakeFieldMapper f1 = new FakeFieldMapper("foo");
-        FakeFieldMapper f2 = new FakeFieldMapper("bar");
+        MockFieldMapper f1 = new MockFieldMapper("foo");
+        MockFieldMapper f2 = new MockFieldMapper("bar");
         FieldTypeLookup lookup = new FieldTypeLookup();
         lookup = lookup.copyAndAddAll("type", newList(f1, f2), randomBoolean());
         Collection<String> names = lookup.simpleMatchToFullName("b*");
@@ -175,7 +174,7 @@ public class FieldTypeLookupTests extends ESTestCase {
     }
 
     public void testIteratorImmutable() {
-        FakeFieldMapper f1 = new FakeFieldMapper("foo");
+        MockFieldMapper f1 = new MockFieldMapper("foo");
         FieldTypeLookup lookup = new FieldTypeLookup();
         lookup = lookup.copyAndAddAll("type", newList(f1), randomBoolean());
 
@@ -194,59 +193,6 @@ public class FieldTypeLookupTests extends ESTestCase {
         return Arrays.asList(mapper);
     }
 
-    // this sucks how much must be overridden just do get a dummy field mapper...
-    static class FakeFieldMapper extends FieldMapper {
-        static Settings dummySettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
-        public FakeFieldMapper(String fullName) {
-            super(fullName, makeFieldType(fullName), makeFieldType(fullName), dummySettings, null, null);
-        }
-        public FakeFieldMapper(String fullName, MappedFieldType fieldType) {
-            super(fullName, fieldType, fieldType, dummySettings, null, null);
-        }
-        static MappedFieldType makeFieldType(String fullName) {
-            FakeFieldType fieldType = new FakeFieldType();
-            fieldType.setName(fullName);
-            return fieldType;
-        }
-        static MappedFieldType makeOtherFieldType(String fullName) {
-            OtherFakeFieldType fieldType = new OtherFakeFieldType();
-            fieldType.setName(fullName);
-            return fieldType;
-        }
-        static class FakeFieldType extends MappedFieldType {
-            public FakeFieldType() {}
-            protected FakeFieldType(FakeFieldType ref) {
-                super(ref);
-            }
-            @Override
-            public MappedFieldType clone() {
-                return new FakeFieldType(this);
-            }
-            @Override
-            public String typeName() {
-                return "faketype";
-            }
-        }
-        static class OtherFakeFieldType extends MappedFieldType {
-            public OtherFakeFieldType() {}
-            protected OtherFakeFieldType(OtherFakeFieldType ref) {
-                super(ref);
-            }
-            @Override
-            public MappedFieldType clone() {
-                return new OtherFakeFieldType(this);
-            }
-            @Override
-            public String typeName() {
-                return "otherfaketype";
-            }
-        }
-        @Override
-        protected String contentType() { return null; }
-        @Override
-        protected void parseCreateField(ParseContext context, List list) throws IOException {}
-    }
-
     private int size(Iterator<MappedFieldType> iterator) {
         if (iterator == null) {
             throw new NullPointerException("iterator");
@@ -258,4 +204,23 @@ public class FieldTypeLookupTests extends ESTestCase {
         }
         return count;
     }
+
+    static class OtherFakeFieldType extends MappedFieldType {
+        public OtherFakeFieldType() {
+        }
+
+        protected OtherFakeFieldType(OtherFakeFieldType ref) {
+            super(ref);
+        }
+
+        @Override
+        public MappedFieldType clone() {
+            return new OtherFakeFieldType(this);
+        }
+
+        @Override
+        public String typeName() {
+            return "otherfaketype";
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
index 966edf82621..d6a161f9ea1 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.similarity.BM25SimilarityProvider;
 import org.elasticsearch.test.ESTestCase;
 
@@ -130,16 +129,10 @@ public abstract class FieldTypeTestCase extends ESTestCase {
                 other.setSimilarity(new BM25SimilarityProvider("bar", Settings.EMPTY));
             }
         },
-        new Modifier("norms.loading", true) {
+        new Modifier("eager_global_ordinals", true) {
             @Override
             public void modify(MappedFieldType ft) {
-                ft.setNormsLoading(MappedFieldType.Loading.LAZY);
-            }
-        },
-        new Modifier("fielddata", true) {
-            @Override
-            public void modify(MappedFieldType ft) {
-                ft.setFieldDataType(new FieldDataType("foo", Settings.builder().put("loading", "eager").build()));
+                ft.setEagerGlobalOrdinals(ft.eagerGlobalOrdinals() == false);
             }
         },
         new Modifier("null_value", true) {
@@ -217,8 +210,7 @@ public abstract class FieldTypeTestCase extends ESTestCase {
             ", searchAnalyzer=" + ft.searchAnalyzer() +
             ", searchQuoteAnalyzer=" + ft.searchQuoteAnalyzer() +
             ", similarity=" + ft.similarity() +
-            ", normsLoading=" + ft.normsLoading() +
-            ", fieldDataType=" + ft.fieldDataType() +
+            ", eagerGlobalOrdinals=" + ft.eagerGlobalOrdinals() +
             ", nullValue=" + ft.nullValue() +
             ", nullValueAsString='" + ft.nullValueAsString() + "'" +
             "} " + super.toString();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
index 191ce5d477e..501c538b870 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
@@ -223,7 +223,7 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase {
     }
 
     public void testRandom() throws Exception {
-        boolean omitNorms = false;
+        boolean norms = true;
         boolean stored = false;
         boolean enabled = true;
         boolean tv_stored = false;
@@ -239,7 +239,7 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase {
             allDefault = false;
             mappingBuilder.startObject("_all");
             if (randomBoolean()) {
-                booleanOptionList.add(new Tuple<>("omit_norms", omitNorms = randomBoolean()));
+                booleanOptionList.add(new Tuple<>("norms", norms = randomBoolean()));
             }
             if (randomBoolean()) {
                 booleanOptionList.add(new Tuple<>("store", stored = randomBoolean()));
@@ -272,7 +272,7 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase {
 
         DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
         String mapping = mappingBuilder.endObject().endObject().bytes().toUtf8();
-        logger.info(mapping);
+        logger.info("Mapping: {}", mapping);
         DocumentMapper docMapper = parser.parse("test", new CompressedXContent(mapping));
         String builtMapping = docMapper.mappingSource().string();
         // reparse it
@@ -285,7 +285,7 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase {
         Document doc = builtDocMapper.parse("test", "test", "1", new BytesArray(json)).rootDoc();
         AllField field = (AllField) doc.getField("_all");
         if (enabled) {
-            assertThat(field.fieldType().omitNorms(), equalTo(omitNorms));
+            assertThat(field.fieldType().omitNorms(), equalTo(!norms));
             assertThat(field.fieldType().stored(), equalTo(stored));
             assertThat(field.fieldType().storeTermVectorOffsets(), equalTo(tv_offsets));
             assertThat(field.fieldType().storeTermVectorPayloads(), equalTo(tv_payloads));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/boost/CustomBoostMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/boost/CustomBoostMappingTests.java
index 91a0ca15cd4..490477d67e7 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/boost/CustomBoostMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/boost/CustomBoostMappingTests.java
@@ -85,18 +85,17 @@ public class CustomBoostMappingTests extends ESSingleNodeTestCase {
     }
 
     public void testBackCompatFieldMappingBoostValues() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
-            .startObject("s_field").field("type", "keyword").field("boost", 2.0f).endObject()
-            .startObject("l_field").field("type", "long").field("boost", 3.0f).startObject("norms").field("enabled", true).endObject().endObject()
-            .startObject("i_field").field("type", "integer").field("boost", 4.0f).startObject("norms").field("enabled", true).endObject().endObject()
-            .startObject("sh_field").field("type", "short").field("boost", 5.0f).startObject("norms").field("enabled", true).endObject().endObject()
-            .startObject("b_field").field("type", "byte").field("boost", 6.0f).startObject("norms").field("enabled", true).endObject().endObject()
-            .startObject("d_field").field("type", "double").field("boost", 7.0f).startObject("norms").field("enabled", true).endObject().endObject()
-            .startObject("f_field").field("type", "float").field("boost", 8.0f).startObject("norms").field("enabled", true).endObject().endObject()
-            .startObject("date_field").field("type", "date").field("boost", 9.0f).startObject("norms").field("enabled", true).endObject().endObject()
-            .endObject().endObject().endObject().string();
-
         {
+            String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
+                    .startObject("s_field").field("type", "keyword").field("boost", 2.0f).endObject()
+                    .startObject("l_field").field("type", "long").field("boost", 3.0f).endObject()
+                    .startObject("i_field").field("type", "integer").field("boost", 4.0f).endObject()
+                    .startObject("sh_field").field("type", "short").field("boost", 5.0f).endObject()
+                    .startObject("b_field").field("type", "byte").field("boost", 6.0f).endObject()
+                    .startObject("d_field").field("type", "double").field("boost", 7.0f).endObject()
+                    .startObject("f_field").field("type", "float").field("boost", 8.0f).endObject()
+                    .startObject("date_field").field("type", "date").field("boost", 9.0f).endObject()
+                    .endObject().endObject().endObject().string();
             IndexService indexService = createIndex("test", BW_SETTINGS);
             QueryShardContext context = indexService.newQueryShardContext();
             DocumentMapper mapper = indexService.mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
@@ -122,16 +121,34 @@ public class CustomBoostMappingTests extends ESSingleNodeTestCase {
                 .endObject().bytes());
 
             assertThat(doc.rootDoc().getField("s_field").boost(), equalTo(2.0f));
+            assertThat(doc.rootDoc().getField("s_field").fieldType().omitNorms(), equalTo(false));
             assertThat(doc.rootDoc().getField("l_field").boost(), equalTo(3.0f));
+            assertThat(doc.rootDoc().getField("l_field").fieldType().omitNorms(), equalTo(false));
             assertThat(doc.rootDoc().getField("i_field").boost(), equalTo(4.0f));
+            assertThat(doc.rootDoc().getField("i_field").fieldType().omitNorms(), equalTo(false));
             assertThat(doc.rootDoc().getField("sh_field").boost(), equalTo(5.0f));
+            assertThat(doc.rootDoc().getField("sh_field").fieldType().omitNorms(), equalTo(false));
             assertThat(doc.rootDoc().getField("b_field").boost(), equalTo(6.0f));
+            assertThat(doc.rootDoc().getField("b_field").fieldType().omitNorms(), equalTo(false));
             assertThat(doc.rootDoc().getField("d_field").boost(), equalTo(7.0f));
+            assertThat(doc.rootDoc().getField("d_field").fieldType().omitNorms(), equalTo(false));
             assertThat(doc.rootDoc().getField("f_field").boost(), equalTo(8.0f));
+            assertThat(doc.rootDoc().getField("f_field").fieldType().omitNorms(), equalTo(false));
             assertThat(doc.rootDoc().getField("date_field").boost(), equalTo(9.0f));
+            assertThat(doc.rootDoc().getField("date_field").fieldType().omitNorms(), equalTo(false));
         }
 
         {
+            String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
+                    .startObject("s_field").field("type", "keyword").field("boost", 2.0f).endObject()
+                    .startObject("l_field").field("type", "long").field("boost", 3.0f).endObject()
+                    .startObject("i_field").field("type", "integer").field("boost", 4.0f).endObject()
+                    .startObject("sh_field").field("type", "short").field("boost", 5.0f).endObject()
+                    .startObject("b_field").field("type", "byte").field("boost", 6.0f).endObject()
+                    .startObject("d_field").field("type", "double").field("boost", 7.0f).endObject()
+                    .startObject("f_field").field("type", "float").field("boost", 8.0f).endObject()
+                    .startObject("date_field").field("type", "date").field("boost", 9.0f).endObject()
+                    .endObject().endObject().endObject().string();
             IndexService indexService = createIndex("text");
             QueryShardContext context = indexService.newQueryShardContext();
             DocumentMapper mapper = indexService.mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
@@ -157,13 +174,21 @@ public class CustomBoostMappingTests extends ESSingleNodeTestCase {
                 .endObject().bytes());
 
             assertThat(doc.rootDoc().getField("s_field").boost(), equalTo(1f));
+            assertThat(doc.rootDoc().getField("s_field").fieldType().omitNorms(), equalTo(true));
             assertThat(doc.rootDoc().getField("l_field").boost(), equalTo(1f));
+            assertThat(doc.rootDoc().getField("l_field").fieldType().omitNorms(), equalTo(true));
             assertThat(doc.rootDoc().getField("i_field").boost(), equalTo(1f));
+            assertThat(doc.rootDoc().getField("i_field").fieldType().omitNorms(), equalTo(true));
             assertThat(doc.rootDoc().getField("sh_field").boost(), equalTo(1f));
+            assertThat(doc.rootDoc().getField("sh_field").fieldType().omitNorms(), equalTo(true));
             assertThat(doc.rootDoc().getField("b_field").boost(), equalTo(1f));
+            assertThat(doc.rootDoc().getField("b_field").fieldType().omitNorms(), equalTo(true));
             assertThat(doc.rootDoc().getField("d_field").boost(), equalTo(1f));
+            assertThat(doc.rootDoc().getField("d_field").fieldType().omitNorms(), equalTo(true));
             assertThat(doc.rootDoc().getField("f_field").boost(), equalTo(1f));
+            assertThat(doc.rootDoc().getField("f_field").fieldType().omitNorms(), equalTo(true));
             assertThat(doc.rootDoc().getField("date_field").boost(), equalTo(1f));
+            assertThat(doc.rootDoc().getField("date_field").fieldType().omitNorms(), equalTo(true));
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/boost/FieldLevelBoostTests.java b/core/src/test/java/org/elasticsearch/index/mapper/boost/FieldLevelBoostTests.java
index b1fde6bdd67..90121e66ea8 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/boost/FieldLevelBoostTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/boost/FieldLevelBoostTests.java
@@ -102,13 +102,13 @@ public class FieldLevelBoostTests extends ESSingleNodeTestCase {
     public void testBackCompatFieldLevelMappingBoost() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("person").startObject("properties")
             .startObject("str_field").field("type", "keyword").field("boost", "2.0").endObject()
-            .startObject("int_field").field("type", "integer").field("boost", "3.0").startObject("norms").field("enabled", true).endObject().endObject()
-            .startObject("byte_field").field("type", "byte").field("boost", "4.0").startObject("norms").field("enabled", true).endObject().endObject()
-            .startObject("date_field").field("type", "date").field("boost", "5.0").startObject("norms").field("enabled", true).endObject().endObject()
-            .startObject("double_field").field("type", "double").field("boost", "6.0").startObject("norms").field("enabled", true).endObject().endObject()
-            .startObject("float_field").field("type", "float").field("boost", "7.0").startObject("norms").field("enabled", true).endObject().endObject()
-            .startObject("long_field").field("type", "long").field("boost", "8.0").startObject("norms").field("enabled", true).endObject().endObject()
-            .startObject("short_field").field("type", "short").field("boost", "9.0").startObject("norms").field("enabled", true).endObject().endObject()
+            .startObject("int_field").field("type", "integer").field("boost", "3.0").endObject()
+            .startObject("byte_field").field("type", "byte").field("boost", "4.0").endObject()
+            .startObject("date_field").field("type", "date").field("boost", "5.0").endObject()
+            .startObject("double_field").field("type", "double").field("boost", "6.0").endObject()
+            .startObject("float_field").field("type", "float").field("boost", "7.0").endObject()
+            .startObject("long_field").field("type", "long").field("boost", "8.0").endObject()
+            .startObject("short_field").field("type", "short").field("boost", "9.0").endObject()
             .string();
 
         {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/KeywordFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/KeywordFieldMapperTests.java
index 8af92f266a5..28867ed1f73 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/core/KeywordFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/KeywordFieldMapperTests.java
@@ -24,22 +24,33 @@ import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.IndexableFieldType;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collection;
 
 import static org.hamcrest.Matchers.equalTo;
 
 public class KeywordFieldMapperTests extends ESSingleNodeTestCase {
 
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(InternalSettingsPlugin.class);
+    }
+
     IndexService indexService;
     DocumentMapperParser parser;
 
@@ -232,4 +243,51 @@ public class KeywordFieldMapperTests extends ESSingleNodeTestCase {
             assertEquals("The [keyword] field does not support positions, got [index_options]=" + indexOptions, e.getMessage());
         }
     }
+
+    public void testBoost() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", "keyword").field("boost", 2f).endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+    }
+
+    public void testBoostImplicitlyEnablesNormsOnOldIndex() throws IOException {
+        indexService = createIndex("test2",
+                Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build());
+        parser = indexService.mapperService().documentMapperParser();
+
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", "keyword").field("boost", 2f).endObject().endObject()
+                .endObject().endObject().string();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        String expectedMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", "keyword")
+                .field("boost", 2f).field("norms", true).endObject().endObject()
+                .endObject().endObject().string();
+        assertEquals(expectedMapping, mapper.mappingSource().toString());
+    }
+
+    public void testEnableNorms() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", "keyword").field("norms", true).endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("field", "1234")
+                .endObject()
+                .bytes());
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+        assertFalse(fields[0].fieldType().omitNorms());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/StringFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/StringFieldTypeTests.java
index 1280fd4acca..49b19d258f6 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/core/StringFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/StringFieldTypeTests.java
@@ -20,10 +20,43 @@ package org.elasticsearch.index.mapper.core;
 
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.FieldTypeTestCase.Modifier;
+import org.junit.Before;
 
 public class StringFieldTypeTests extends FieldTypeTestCase {
     @Override
     protected MappedFieldType createDefaultFieldType() {
         return new StringFieldMapper.StringFieldType();
     }
+    @Before
+    public void setupProperties() {
+        addModifier(new Modifier("fielddata", true) {
+            @Override
+            public void modify(MappedFieldType ft) {
+                StringFieldMapper.StringFieldType tft = (StringFieldMapper.StringFieldType)ft;
+                tft.setFielddata(tft.fielddata() == false);
+            }
+        });
+        addModifier(new Modifier("fielddata_frequency_filter.min", true) {
+            @Override
+            public void modify(MappedFieldType ft) {
+                StringFieldMapper.StringFieldType tft = (StringFieldMapper.StringFieldType)ft;
+                tft.setFielddataMinFrequency(3);
+            }
+        });
+        addModifier(new Modifier("fielddata_frequency_filter.max", true) {
+            @Override
+            public void modify(MappedFieldType ft) {
+                StringFieldMapper.StringFieldType tft = (StringFieldMapper.StringFieldType)ft;
+                tft.setFielddataMaxFrequency(0.2);
+            }
+        });
+        addModifier(new Modifier("fielddata_frequency_filter.min_segment_size", true) {
+            @Override
+            public void modify(MappedFieldType ft) {
+                StringFieldMapper.StringFieldType tft = (StringFieldMapper.StringFieldType)ft;
+                tft.setFielddataMinSegmentSize(1000);
+            }
+        });
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java
new file mode 100644
index 00000000000..92cee7c5407
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java
@@ -0,0 +1,350 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper.core;
+
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+
+import org.apache.lucene.index.IndexOptions;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.DocumentMapperParser;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.core.TextFieldMapper.TextFieldType;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class StringMappingUpgradeTests extends ESSingleNodeTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(InternalSettingsPlugin.class);
+    }
+
+    public void testUpgradeDefaults() throws IOException {
+        IndexService indexService = createIndex("test");
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", "string").endObject().endObject()
+                .endObject().endObject().string();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        FieldMapper field = mapper.mappers().getMapper("field");
+        assertThat(field, instanceOf(TextFieldMapper.class));
+    }
+
+    public void testUpgradeAnalyzedString() throws IOException {
+        IndexService indexService = createIndex("test");
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", "string").field("index", "analyzed").endObject().endObject()
+                .endObject().endObject().string();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        FieldMapper field = mapper.mappers().getMapper("field");
+        assertThat(field, instanceOf(TextFieldMapper.class));
+    }
+
+    public void testUpgradeNotAnalyzedString() throws IOException {
+        IndexService indexService = createIndex("test");
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", "string")
+                .field("index", "not_analyzed").endObject().endObject()
+                .endObject().endObject().string();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        FieldMapper field = mapper.mappers().getMapper("field");
+        assertThat(field, instanceOf(KeywordFieldMapper.class));
+    }
+
+    public void testUpgradeNotIndexedString() throws IOException {
+        IndexService indexService = createIndex("test");
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", "string").field("index", "no").endObject().endObject()
+                .endObject().endObject().string();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        FieldMapper field = mapper.mappers().getMapper("field");
+        assertThat(field, instanceOf(KeywordFieldMapper.class));
+        assertEquals(IndexOptions.NONE, field.fieldType().indexOptions());
+    }
+
+    public void testNotSupportedUpgrade() throws IOException {
+        IndexService indexService = createIndex("test");
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", "string").field("analyzer", "keyword").endObject().endObject()
+                .endObject().endObject().string();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> parser.parse("type", new CompressedXContent(mapping)));
+        assertThat(e.getMessage(), containsString("The [string] type is removed in 5.0"));
+    }
+
+    public void testUpgradeFielddataSettings() throws IOException {
+        IndexService indexService = createIndex("test");
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        String format = randomFrom("paged_bytes", "disabled");
+        String loading = randomFrom("lazy", "eager", "eager_global_ordinals");
+        boolean keyword = random().nextBoolean();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties")
+                    .startObject("field")
+                        .field("type", "string")
+                        .field("index", keyword ? "not_analyzed" : "analyzed")
+                        .startObject("fielddata")
+                            .field("format", format)
+                            .field("loading", loading)
+                            .startObject("filter")
+                                .startObject("frequency")
+                                    .field("min", 3)
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endObject()
+                .endObject().endObject().string();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        FieldMapper field = mapper.mappers().getMapper("field");
+        if (keyword) {
+            assertThat(field, instanceOf(KeywordFieldMapper.class));
+        } else {
+            assertThat(field, instanceOf(TextFieldMapper.class));
+            TextFieldType fieldType = (TextFieldType) field.fieldType();
+            assertEquals("disabled".equals(format) == false, fieldType.fielddata());
+            assertEquals(3, fieldType.fielddataMinFrequency(), 0d);
+            assertEquals(Integer.MAX_VALUE, fieldType.fielddataMaxFrequency(), 0d);
+        }
+        assertEquals("eager_global_ordinals".equals(loading), field.fieldType().eagerGlobalOrdinals());
+    }
+
+    public void testUpgradeIgnoreAbove() throws IOException {
+        IndexService indexService = createIndex("test");
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", "string")
+                .field("index", "not_analyzed").field("ignore_above", 200).endObject().endObject()
+                .endObject().endObject().string();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        FieldMapper field = mapper.mappers().getMapper("field");
+        assertThat(field, instanceOf(KeywordFieldMapper.class));
+        assertEquals(200, ((KeywordFieldMapper) field).ignoreAbove());
+    }
+
+    public void testUpgradeRandomMapping() throws IOException {
+        final int iters = 20;
+        for (int i = 0; i < iters; ++i) {
+            doTestUpgradeRandomMapping(i);
+        }
+    }
+
+    private void doTestUpgradeRandomMapping(int iter) throws IOException {
+        IndexService indexService;
+        boolean oldIndex = randomBoolean();
+        String indexName = "test" + iter;
+        if (oldIndex) {
+            Settings settings = Settings.builder()
+                    .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0)
+                    .build();
+            indexService = createIndex(indexName, settings);
+        } else {
+            indexService = createIndex(indexName);
+        }
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", "string");
+        boolean keyword = randomBoolean();
+        boolean hasNorms = keyword == false;
+        boolean shouldUpgrade = true;
+        if (keyword) {
+            mapping.field("index", randomBoolean() ? "not_analyzed" : "no");
+        } else if (randomBoolean()) {
+            mapping.field("index", "analyzed");
+        }
+        if (randomBoolean()) {
+            mapping.field("store", RandomPicks.randomFrom(random(), Arrays.asList("yes", "no", true, false)));
+        }
+        if (keyword && randomBoolean()) {
+            mapping.field("doc_values", randomBoolean());
+        }
+        if (randomBoolean()) {
+            hasNorms = randomBoolean();
+            if (randomBoolean()) {
+                mapping.field("omit_norms", hasNorms == false);
+            } else {
+                mapping.field("norms", Collections.singletonMap("enabled", hasNorms));
+            }
+        }
+        if (randomBoolean()) {
+            Map<String, Object> fielddata = new HashMap<>();
+            if (randomBoolean()) {
+                fielddata.put("format", randomFrom("paged_bytes", "disabled"));
+            }
+            if (randomBoolean()) {
+                fielddata.put("loading", randomFrom("lazy", "eager", "eager_global_ordinals"));
+            }
+            if (randomBoolean()) {
+                Map<String, Object> frequencyFilter = new HashMap<>();
+                frequencyFilter.put("min", 10);
+                frequencyFilter.put("max", 1000);
+                frequencyFilter.put("min_segment_size", 10000);
+            }
+        }
+        if (randomBoolean()) {
+            mapping.startObject("fields").startObject("raw").field("type", "keyword").endObject().endObject();
+        }
+        if (randomBoolean()) {
+            mapping.field("copy_to", "bar");
+        }
+        if (randomBoolean()) {
+            // this option is not upgraded automatically
+            mapping.field("index_options", "docs");
+            shouldUpgrade = false;
+        }
+        mapping.endObject().endObject().endObject().endObject();
+
+        if (oldIndex == false && shouldUpgrade == false) {
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                    () -> parser.parse("type", new CompressedXContent(mapping.string())));
+            assertThat(e.getMessage(), containsString("The [string] type is removed in 5.0"));
+        } else {
+            DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping.string()));
+            FieldMapper field = mapper.mappers().getMapper("field");
+            if (oldIndex) {
+                assertThat(field, instanceOf(StringFieldMapper.class));
+            } else if (keyword) {
+                assertThat(field, instanceOf(KeywordFieldMapper.class));
+            } else {
+                assertThat(field, instanceOf(TextFieldMapper.class));
+            }
+            if (field.fieldType().indexOptions() != IndexOptions.NONE) {
+                assertEquals(hasNorms, field.fieldType().omitNorms() == false);
+            }
+        }
+    }
+
+    public void testUpgradeTemplateWithDynamicType() throws IOException {
+        IndexService indexService = createIndex("test");
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startArray("dynamic_templates")
+                    .startObject()
+                        .startObject("my_template")
+                            .field("match_mapping_type", "string")
+                            .startObject("mapping")
+                                .field("store", true)
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endArray()
+                .endObject().endObject().string();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        BytesReference source = XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject().bytes();
+        ParsedDocument doc = mapper.parse("test", "type", "id", source);
+        Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo");
+        assertThat(fooMapper, instanceOf(TextFieldMapper.class));
+        assertTrue(((TextFieldMapper) fooMapper).fieldType().stored());
+    }
+
+    public void testUpgradeTemplateWithDynamicType2() throws IOException {
+        IndexService indexService = createIndex("test");
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startArray("dynamic_templates")
+                    .startObject()
+                        .startObject("my_template")
+                            .field("match_mapping_type", "string")
+                            .startObject("mapping")
+                                .field("type", "{dynamic_type}")
+                                .field("store", true)
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endArray()
+                .endObject().endObject().string();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        BytesReference source = XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject().bytes();
+        ParsedDocument doc = mapper.parse("test", "type", "id", source);
+        Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo");
+        assertThat(fooMapper, instanceOf(TextFieldMapper.class));
+        assertTrue(((TextFieldMapper) fooMapper).fieldType().stored());
+    }
+
+    public void testUpgradeTemplateWithDynamicTypeKeyword() throws IOException {
+        IndexService indexService = createIndex("test");
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startArray("dynamic_templates")
+                    .startObject()
+                        .startObject("my_template")
+                            .field("match_mapping_type", "string")
+                            .startObject("mapping")
+                                .field("index", "not_analyzed")
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endArray()
+                .endObject().endObject().string();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        BytesReference source = XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject().bytes();
+        ParsedDocument doc = mapper.parse("test", "type", "id", source);
+        Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo");
+        assertThat(fooMapper, instanceOf(KeywordFieldMapper.class));
+    }
+
+    public void testUpgradeTemplateWithDynamicTypeKeyword2() throws IOException {
+        IndexService indexService = createIndex("test");
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startArray("dynamic_templates")
+                    .startObject()
+                        .startObject("my_template")
+                            .field("match_mapping_type", "string")
+                            .startObject("mapping")
+                                .field("type", "{dynamic_type}")
+                                .field("index", "not_analyzed")
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endArray()
+                .endObject().endObject().string();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        BytesReference source = XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject().bytes();
+        ParsedDocument doc = mapper.parse("test", "type", "id", source);
+        Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo");
+        assertThat(fooMapper, instanceOf(KeywordFieldMapper.class));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/TextFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/TextFieldMapperTests.java
index 3a9d5b46ab9..1581727937a 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/core/TextFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/TextFieldMapperTests.java
@@ -36,6 +36,7 @@ import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
+import org.elasticsearch.index.mapper.core.TextFieldMapper.TextFieldType;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -45,6 +46,7 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
 public class TextFieldMapperTests extends ESSingleNodeTestCase {
@@ -132,9 +134,7 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("field")
                     .field("type", "text")
-                    .startObject("norms")
-                        .field("enabled", false)
-                    .endObject()
+                    .field("norms", false)
                 .endObject().endObject()
                 .endObject().endObject().string();
 
@@ -386,4 +386,54 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
         assertThat(doc.rootDoc().getField("field6").fieldType().storeTermVectorPositions(), equalTo(true));
         assertThat(doc.rootDoc().getField("field6").fieldType().storeTermVectorPayloads(), equalTo(true));
     }
+
+    public void testEagerGlobalOrdinals() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field")
+                    .field("type", "text")
+                    .field("eager_global_ordinals", true)
+                .endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+        assertTrue(mapper.mappers().getMapper("field").fieldType().eagerGlobalOrdinals());
+    }
+
+    public void testFielddata() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field")
+                    .field("type", "text")
+                    .field("fielddata", false)
+                .endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+        IllegalStateException e = expectThrows(IllegalStateException.class,
+                () -> mapper.mappers().getMapper("field").fieldType().fielddataBuilder());
+        assertThat(e.getMessage(), containsString("Fielddata is disabled"));
+    }
+
+    public void testFrequencyFilter() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field")
+                    .field("type", "text")
+                    .startObject("fielddata_frequency_filter")
+                        .field("min", 2d)
+                        .field("min_segment_size", 1000)
+                    .endObject()
+                .endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+        TextFieldType fieldType = (TextFieldType) mapper.mappers().getMapper("field").fieldType();
+        assertThat(fieldType.fielddataMinFrequency(), equalTo(2d));
+        assertThat(fieldType.fielddataMaxFrequency(), equalTo((double) Integer.MAX_VALUE));
+        assertThat(fieldType.fielddataMinSegmentSize(), equalTo(1000));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/TextFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/TextFieldTypeTests.java
index a6c453c4a3c..ea65bfc3421 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/core/TextFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/TextFieldTypeTests.java
@@ -20,10 +20,43 @@ package org.elasticsearch.index.mapper.core;
 
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.junit.Before;
 
 public class TextFieldTypeTests extends FieldTypeTestCase {
     @Override
     protected MappedFieldType createDefaultFieldType() {
         return new TextFieldMapper.TextFieldType();
     }
+
+    @Before
+    public void setupProperties() {
+        addModifier(new Modifier("fielddata", true) {
+            @Override
+            public void modify(MappedFieldType ft) {
+                TextFieldMapper.TextFieldType tft = (TextFieldMapper.TextFieldType)ft;
+                tft.setFielddata(tft.fielddata() == false);
+            }
+        });
+        addModifier(new Modifier("fielddata_frequency_filter.min", true) {
+            @Override
+            public void modify(MappedFieldType ft) {
+                TextFieldMapper.TextFieldType tft = (TextFieldMapper.TextFieldType)ft;
+                tft.setFielddataMinFrequency(3);
+            }
+        });
+        addModifier(new Modifier("fielddata_frequency_filter.max", true) {
+            @Override
+            public void modify(MappedFieldType ft) {
+                TextFieldMapper.TextFieldType tft = (TextFieldMapper.TextFieldType)ft;
+                tft.setFielddataMaxFrequency(0.2);
+            }
+        });
+        addModifier(new Modifier("fielddata_frequency_filter.min_segment_size", true) {
+            @Override
+            public void modify(MappedFieldType ft) {
+                TextFieldMapper.TextFieldType tft = (TextFieldMapper.TextFieldType)ft;
+                tft.setFielddataMinSegmentSize(1000);
+            }
+        });
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java
index c0eca41b26f..31e4e2d0923 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java
@@ -128,9 +128,7 @@ public class TokenCountFieldMapperIntegrationIT extends ESIntegTestCase {
                                 .startObject("token_count_with_doc_values")
                                     .field("type", "token_count")
                                     .field("analyzer", "standard")
-                                    .startObject("fielddata")
-                                        .field("format", "doc_values")
-                                    .endObject()
+                                    .field("doc_values", true)
                                 .endObject()
                             .endObject()
                         .endObject()
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java
index 8c25713ce3d..e1b94967b43 100755
--- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java
@@ -125,16 +125,6 @@ public class ExternalMapper extends FieldMapper {
         public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             ExternalMapper.Builder builder = new ExternalMapper.Builder(name, generatedValue, mapperName);
             parseField(builder, name, node, parserContext);
-            for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
-                Map.Entry<String, Object> entry = iterator.next();
-                String propName = Strings.toUnderscoreCase(entry.getKey());
-                Object propNode = entry.getValue();
-
-                if (parseMultiField(builder, name, parserContext, propName, propNode)) {
-                    iterator.remove();
-                }
-            }
-
             return builder;
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldTypeTests.java
index fd0c344c26b..f6ed0cf931d 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldTypeTests.java
@@ -18,6 +18,9 @@
  */
 package org.elasticsearch.index.mapper.internal;
 
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.junit.Before;
@@ -38,4 +41,15 @@ public class FieldNamesFieldTypeTests extends FieldTypeTestCase {
             }
         });
     }
+
+    public void testTermQuery() {
+        FieldNamesFieldMapper.FieldNamesFieldType type = new FieldNamesFieldMapper.FieldNamesFieldType();
+        type.setName(FieldNamesFieldMapper.CONTENT_TYPE);
+        type.setEnabled(true);
+        Query termQuery = type.termQuery("field_name", null);
+        assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.CONTENT_TYPE, "field_name")), termQuery);
+        type.setEnabled(false);
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> type.termQuery("field_name", null));
+        assertEquals("Cannot run [exists] queries if the [_field_names] field is disabled", e.getMessage());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/ParentFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/ParentFieldMapperTests.java
index 7083d9fa6bc..9b5504bf70f 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/internal/ParentFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/ParentFieldMapperTests.java
@@ -22,33 +22,28 @@ import org.apache.lucene.index.DocValuesType;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.ContentPath;
-import org.elasticsearch.index.mapper.MappedFieldType.Loading;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.test.ESTestCase;
 
-import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.nullValue;
 
 public class ParentFieldMapperTests extends ESTestCase {
 
     public void testPost2Dot0LazyLoading() {
         ParentFieldMapper.Builder builder = new ParentFieldMapper.Builder("child");
         builder.type("parent");
-        builder.fieldDataSettings(createFDSettings(Loading.LAZY));
+        builder.eagerGlobalOrdinals(false);
 
         ParentFieldMapper parentFieldMapper = builder.build(new Mapper.BuilderContext(post2Dot0IndexSettings(), new ContentPath(0)));
 
         assertThat(parentFieldMapper.getParentJoinFieldType().name(), equalTo("_parent#child"));
-        assertThat(parentFieldMapper.getParentJoinFieldType().fieldDataType(), nullValue());
         assertThat(parentFieldMapper.getParentJoinFieldType().hasDocValues(), is(true));
         assertThat(parentFieldMapper.getParentJoinFieldType().docValuesType(), equalTo(DocValuesType.SORTED));
 
         assertThat(parentFieldMapper.fieldType().name(), equalTo("_parent#parent"));
-        assertThat(parentFieldMapper.fieldType().fieldDataType().getLoading(), equalTo(Loading.LAZY));
+        assertThat(parentFieldMapper.fieldType().eagerGlobalOrdinals(), equalTo(false));
         assertThat(parentFieldMapper.fieldType().hasDocValues(), is(true));
         assertThat(parentFieldMapper.fieldType().docValuesType(), equalTo(DocValuesType.SORTED));
     }
@@ -56,35 +51,16 @@ public class ParentFieldMapperTests extends ESTestCase {
     public void testPost2Dot0EagerLoading() {
         ParentFieldMapper.Builder builder = new ParentFieldMapper.Builder("child");
         builder.type("parent");
-        builder.fieldDataSettings(createFDSettings(Loading.EAGER));
+        builder.eagerGlobalOrdinals(true);
 
         ParentFieldMapper parentFieldMapper = builder.build(new Mapper.BuilderContext(post2Dot0IndexSettings(), new ContentPath(0)));
 
         assertThat(parentFieldMapper.getParentJoinFieldType().name(), equalTo("_parent#child"));
-        assertThat(parentFieldMapper.getParentJoinFieldType().fieldDataType(), nullValue());
         assertThat(parentFieldMapper.getParentJoinFieldType().hasDocValues(), is(true));
         assertThat(parentFieldMapper.getParentJoinFieldType().docValuesType(), equalTo(DocValuesType.SORTED));
 
         assertThat(parentFieldMapper.fieldType().name(), equalTo("_parent#parent"));
-        assertThat(parentFieldMapper.fieldType().fieldDataType().getLoading(), equalTo(Loading.EAGER));
-        assertThat(parentFieldMapper.fieldType().hasDocValues(), is(true));
-        assertThat(parentFieldMapper.fieldType().docValuesType(), equalTo(DocValuesType.SORTED));
-    }
-
-    public void testPost2Dot0EagerGlobalOrdinalsLoading() {
-        ParentFieldMapper.Builder builder = new ParentFieldMapper.Builder("child");
-        builder.type("parent");
-        builder.fieldDataSettings(createFDSettings(Loading.EAGER_GLOBAL_ORDINALS));
-
-        ParentFieldMapper parentFieldMapper = builder.build(new Mapper.BuilderContext(post2Dot0IndexSettings(), new ContentPath(0)));
-
-        assertThat(parentFieldMapper.getParentJoinFieldType().name(), equalTo("_parent#child"));
-        assertThat(parentFieldMapper.getParentJoinFieldType().fieldDataType(), nullValue());
-        assertThat(parentFieldMapper.getParentJoinFieldType().hasDocValues(), is(true));
-        assertThat(parentFieldMapper.getParentJoinFieldType().docValuesType(), equalTo(DocValuesType.SORTED));
-
-        assertThat(parentFieldMapper.fieldType().name(), equalTo("_parent#parent"));
-        assertThat(parentFieldMapper.fieldType().fieldDataType().getLoading(), equalTo(Loading.EAGER_GLOBAL_ORDINALS));
+        assertThat(parentFieldMapper.fieldType().eagerGlobalOrdinals(), equalTo(true));
         assertThat(parentFieldMapper.fieldType().hasDocValues(), is(true));
         assertThat(parentFieldMapper.fieldType().docValuesType(), equalTo(DocValuesType.SORTED));
     }
@@ -93,8 +69,4 @@ public class ParentFieldMapperTests extends ESTestCase {
         return Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_1_0).build();
     }
 
-    private static Settings createFDSettings(Loading loading) {
-        return new FieldDataType("child", settingsBuilder().put(Loading.KEY, loading)).getSettings();
-    }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldTests.java b/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldTests.java
index 38b13baebba..9d33a1129f5 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldTests.java
@@ -30,13 +30,11 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.index.mapper.core.DateFieldMapper;
 import org.elasticsearch.index.mapper.core.KeywordFieldMapper;
-import org.elasticsearch.index.mapper.core.StringFieldMapper;
 import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.index.mapper.core.TokenCountFieldMapper;
 import org.elasticsearch.index.mapper.object.RootObjectMapper;
@@ -118,7 +116,7 @@ public class MultiFieldTests extends ESSingleNodeTestCase {
         assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name.test1").fieldType().indexOptions());
         assertThat(docMapper.mappers().getMapper("name.test1").fieldType().stored(), equalTo(true));
         assertThat(docMapper.mappers().getMapper("name.test1").fieldType().tokenized(), equalTo(true));
-        assertThat(docMapper.mappers().getMapper("name.test1").fieldType().fieldDataType().getLoading(), equalTo(MappedFieldType.Loading.EAGER));
+        assertThat(docMapper.mappers().getMapper("name.test1").fieldType().eagerGlobalOrdinals(), equalTo(true));
 
         assertThat(docMapper.mappers().getMapper("name.test2"), notNullValue());
         assertThat(docMapper.mappers().getMapper("name.test2"), instanceOf(TokenCountFieldMapper.class));
@@ -204,37 +202,6 @@ public class MultiFieldTests extends ESSingleNodeTestCase {
         }
     }
 
-    // The fielddata settings need to be the same after deserializing/re-serialsing, else unnecessary mapping sync's can be triggered
-    public void testMultiFieldsFieldDataSettingsInConsistentOrder() throws Exception {
-        final String MY_MULTI_FIELD = "multi_field";
-
-        // Possible fielddata settings
-        Map<String, Object> possibleSettings = new TreeMap<String, Object>();
-        possibleSettings.put("filter.frequency.min", 1);
-        possibleSettings.put("filter.frequency.max", 2);
-        possibleSettings.put("filter.regex.pattern", ".*");
-        possibleSettings.put("loading", "eager");
-        possibleSettings.put("foo", "bar");
-        possibleSettings.put("zetting", "zValue");
-        possibleSettings.put("aSetting", "aValue");
-
-        // Generate a mapping with the a random subset of possible fielddata settings
-        XContentBuilder builder = jsonBuilder().startObject().startObject("type").startObject("properties")
-            .startObject("my_field").field("type", "text").startObject("fields").startObject(MY_MULTI_FIELD)
-            .field("type", "text").startObject("fielddata");
-        String[] keys = possibleSettings.keySet().toArray(new String[]{});
-        Collections.shuffle(Arrays.asList(keys), random());
-        for(int i = randomIntBetween(0, possibleSettings.size()-1); i >= 0; --i)
-            builder.field(keys[i], possibleSettings.get(keys[i]));
-        builder.endObject().endObject().endObject().endObject().endObject().endObject().endObject();
-
-        // Check the mapping remains identical when deserialed/re-serialsed
-        final DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
-        DocumentMapper docMapper = parser.parse("type", new CompressedXContent(builder.string()));
-        DocumentMapper docMapper2 = parser.parse("type", docMapper.mappingSource());
-        assertThat(docMapper.mappingSource(), equalTo(docMapper2.mappingSource()));
-    }
-
     public void testObjectFieldNotAllowed() throws Exception {
         String mapping = jsonBuilder().startObject().startObject("type").startObject("properties").startObject("my_field")
             .field("type", "text").startObject("fields").startObject("multi").field("type", "object").endObject().endObject()
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldsIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldsIntegrationIT.java
index 6a82052bfa8..a1f6929fade 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldsIntegrationIT.java
@@ -112,7 +112,7 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase {
         assertThat(mappingMetaData, not(nullValue()));
         Map<String, Object> mappingSource = mappingMetaData.sourceAsMap();
         Map aField = ((Map) XContentMapValues.extractValue("properties.a", mappingSource));
-        logger.info("Keys: " + aField.keySet());
+        logger.info("Keys: {}", aField.keySet());
         assertThat(aField.size(), equalTo(2));
         assertThat(aField.get("type").toString(), equalTo("geo_point"));
         assertThat(aField.get("fields"), notNullValue());
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java b/core/src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java
index 09804f82919..4c80b74a01d 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java
@@ -31,13 +31,14 @@ import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.core.DoubleFieldMapper;
+import org.elasticsearch.index.mapper.core.FloatFieldMapper;
 import org.elasticsearch.index.mapper.core.LongFieldMapper;
 import org.elasticsearch.index.mapper.core.NumberFieldMapper;
 import org.elasticsearch.index.mapper.core.TextFieldMapper;
@@ -89,7 +90,7 @@ public class SimpleNumericTests extends ESSingleNodeTestCase {
         assertThat(mapper, instanceOf(LongFieldMapper.class));
 
         mapper = defaultMapper.mappers().smartNameFieldMapper("s_double");
-        assertThat(mapper, instanceOf(DoubleFieldMapper.class));
+        assertThat(mapper, instanceOf(FloatFieldMapper.class));
     }
 
     public void testNumericDetectionDefault() throws Exception {
@@ -417,15 +418,11 @@ public class SimpleNumericTests extends ESSingleNodeTestCase {
                     .startObject("properties")
                         .startObject("int")
                             .field("type", "integer")
-                            .startObject("fielddata")
-                                .field("format", "doc_values")
-                            .endObject()
+                            .field("doc_values", true)
                         .endObject()
                         .startObject("double")
                             .field("type", "double")
-                            .startObject("fielddata")
-                                .field("format", "doc_values")
-                            .endObject()
+                            .field("doc_values", true)
                         .endObject()
                     .endObject()
                 .endObject()
@@ -478,7 +475,8 @@ public class SimpleNumericTests extends ESSingleNodeTestCase {
         Document luceneDoc = doc.docs().get(0);
 
         assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("long"));
-        assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("double"));
+        assertThat(luceneDoc.getField("double").numericValue(), instanceOf(Float.class));
+        assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_32_BIT, luceneDoc.getField("double"));
         assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("date"));
     }
 
@@ -684,4 +682,46 @@ public class SimpleNumericTests extends ESSingleNodeTestCase {
         parser = createIndex("index2-" + type, oldIndexSettings).mapperService().documentMapperParser();
         parser.parse("type", new CompressedXContent(mappingWithTV)); // no exception
     }
+
+    public void testRejectNorms() throws IOException {
+        // not supported as of 5.0
+        for (String type : Arrays.asList("byte", "short", "integer", "long", "float", "double")) {
+            DocumentMapperParser parser = createIndex("index-" + type).mapperService().documentMapperParser();
+            String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties")
+                    .startObject("foo")
+                        .field("type", type)
+                        .field("norms", random().nextBoolean())
+                    .endObject()
+                .endObject().endObject().endObject().string();
+            MapperParsingException e = expectThrows(MapperParsingException.class, () -> parser.parse("type", new CompressedXContent(mapping)));
+            assertThat(e.getMessage(), containsString("Mapping definition for [foo] has unsupported parameters:  [norms"));
+        }
+    }
+
+    public void testIgnoreFielddata() throws IOException {
+        for (String type : Arrays.asList("byte", "short", "integer", "long", "float", "double")) {
+            Settings oldIndexSettings = Settings.builder()
+                    .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_1_0)
+                    .build();
+            DocumentMapperParser parser = createIndex("index-" + type, oldIndexSettings).mapperService().documentMapperParser();
+            String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties")
+                    .startObject("foo")
+                        .field("type", type)
+                        .startObject("fielddata")
+                            .field("loading", "eager")
+                        .endObject()
+                    .endObject()
+                .endObject().endObject().endObject().string();
+            DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+            String expectedMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties")
+                    .startObject("foo")
+                        .field("type", type)
+                    .endObject()
+                .endObject().endObject().endObject().string();
+            assertEquals(expectedMapping, mapper.mappingSource().string());
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/object/SimpleObjectMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/object/SimpleObjectMappingTests.java
index 96d5559f457..907616712a2 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/object/SimpleObjectMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/object/SimpleObjectMappingTests.java
@@ -28,32 +28,28 @@ import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import static org.hamcrest.Matchers.containsString;
 
-/**
- */
 public class SimpleObjectMappingTests extends ESSingleNodeTestCase {
     public void testDifferentInnerObjectTokenFailure() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .endObject().endObject().string();
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-        try {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
             defaultMapper.parse("test", "type", "1", new BytesArray(" {\n" +
-                    "      \"object\": {\n" +
-                    "        \"array\":[\n" +
-                    "        {\n" +
-                    "          \"object\": { \"value\": \"value\" }\n" +
-                    "        },\n" +
-                    "        {\n" +
-                    "          \"object\":\"value\"\n" +
-                    "        }\n" +
-                    "        ]\n" +
-                    "      },\n" +
-                    "      \"value\":\"value\"\n" +
-                    "    }"));
-            fail();
-        } catch (MapperParsingException e) {
-            // all is well
-        }
+                "      \"object\": {\n" +
+                "        \"array\":[\n" +
+                "        {\n" +
+                "          \"object\": { \"value\": \"value\" }\n" +
+                "        },\n" +
+                "        {\n" +
+                "          \"object\":\"value\"\n" +
+                "        }\n" +
+                "        ]\n" +
+                "      },\n" +
+                "      \"value\":\"value\"\n" +
+                "    }"));
+        });
+        assertTrue(e.getMessage(), e.getMessage().contains("different type"));
     }
 
     public void testEmptyArrayProperties() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/string/SimpleStringMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/string/SimpleStringMappingTests.java
index 86c67db219f..2dbb942f95a 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/string/SimpleStringMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/string/SimpleStringMappingTests.java
@@ -44,6 +44,7 @@ import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.core.StringFieldMapper;
 import org.elasticsearch.index.mapper.core.StringFieldMapper.Builder;
+import org.elasticsearch.index.mapper.core.StringFieldMapper.StringFieldType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -564,7 +565,7 @@ public class SimpleStringMappingTests extends ESSingleNodeTestCase {
             mapperService.merge("type", new CompressedXContent(updatedMapping), MapperService.MergeReason.MAPPING_UPDATE, false);
             fail();
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("different [omit_norms]"));
+            assertThat(e.getMessage(), containsString("different [norms]"));
         }
     }
 
@@ -592,4 +593,88 @@ public class SimpleStringMappingTests extends ESSingleNodeTestCase {
         }
     }
 
+    public void testFielddataLoading() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field")
+                    .field("type", "string")
+                    .startObject("fielddata")
+                        .field("loading", "eager_global_ordinals")
+                    .endObject()
+                .endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        String expectedMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field")
+                .field("type", "string")
+                .field("eager_global_ordinals", true)
+            .endObject().endObject()
+            .endObject().endObject().string();
+
+        assertEquals(expectedMapping, mapper.mappingSource().toString());
+        assertTrue(mapper.mappers().getMapper("field").fieldType().eagerGlobalOrdinals());
+    }
+
+    public void testFielddataFilter() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field")
+                    .field("type", "string")
+                    .startObject("fielddata")
+                        .startObject("filter")
+                            .startObject("frequency")
+                                .field("min", 2d)
+                                .field("min_segment_size", 1000)
+                            .endObject()
+                            .startObject("regex")
+                                .field("pattern", "^#.*")
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        String expectedMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field")
+                .field("type", "string")
+                .startObject("fielddata_frequency_filter")
+                    .field("min", 2d)
+                    .field("min_segment_size", 1000)
+                .endObject()
+            .endObject().endObject()
+            .endObject().endObject().string();
+
+        assertEquals(expectedMapping, mapper.mappingSource().toString());
+        StringFieldType fieldType = (StringFieldType) mapper.mappers().getMapper("field").fieldType();
+        assertThat(fieldType.fielddataMinFrequency(), equalTo(2d));
+        assertThat(fieldType.fielddataMaxFrequency(), equalTo((double) Integer.MAX_VALUE));
+        assertThat(fieldType.fielddataMinSegmentSize(), equalTo(1000));
+    }
+
+    public void testDisabledFielddata() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field")
+                    .field("type", "string")
+                    .startObject("fielddata")
+                        .field("format", "disabled")
+                    .endObject()
+                .endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        String expectedMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field")
+                .field("type", "string")
+                .field("fielddata", false)
+            .endObject().endObject()
+            .endObject().endObject().string();
+
+        assertEquals(expectedMapping, mapper.mappingSource().toString());
+        IllegalStateException e = expectThrows(IllegalStateException.class,
+                () -> mapper.mappers().getMapper("field").fieldType().fielddataBuilder());
+        assertThat(e.getMessage(), containsString("Fielddata is disabled"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
index d5efd6dcfc3..0a8d75d42f0 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
@@ -182,7 +182,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
         IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
         request.process(metaData, mappingMetaData, true, "test");
         assertThat(request.timestamp(), notNullValue());
-        assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd"), Version.CURRENT)));
+        assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd"))));
     }
 
     // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java
index a3d6a87c43f..600f84b5f5f 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java
@@ -49,7 +49,7 @@ public class UpdateMappingOnClusterIT extends ESIntegTestCase {
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/all_mapping_create_index.json");
         String mappingUpdate = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json");
         String[] errorMessage = {
-                "[_all] has different [omit_norms] values",
+                "[_all] has different [norms] values",
                 "[_all] has different [store] values",
                 "[_all] has different [store_term_vector] values",
                 "[_all] has different [store_term_vector_offsets] values",
diff --git a/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java b/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java
index f23ec6d9595..572eb1b46c1 100644
--- a/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java
@@ -24,6 +24,8 @@ import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.memory.MemoryIndex;
+import org.apache.lucene.queries.BlendedTermQuery;
+import org.apache.lucene.queries.CommonTermsQuery;
 import org.apache.lucene.queries.TermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
@@ -45,9 +47,7 @@ import java.util.List;
 import java.util.Set;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
 public class ExtractQueryTermsServiceTests extends ESTestCase {
@@ -102,6 +102,27 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         assertThat(terms.get(0).bytes(), equalTo(termQuery.getTerm().bytes()));
     }
 
+    public void testExtractQueryMetadata_termsQuery() {
+        TermsQuery termsQuery = new TermsQuery("_field", new BytesRef("_term1"), new BytesRef("_term2"));
+        List<Term> terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(termsQuery));
+        Collections.sort(terms);
+        assertThat(terms.size(), equalTo(2));
+        assertThat(terms.get(0).field(), equalTo("_field"));
+        assertThat(terms.get(0).text(), equalTo("_term1"));
+        assertThat(terms.get(1).field(), equalTo("_field"));
+        assertThat(terms.get(1).text(), equalTo("_term2"));
+
+        // test with different fields
+        termsQuery = new TermsQuery(new Term("_field1", "_term1"), new Term("_field2", "_term2"));
+        terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(termsQuery));
+        Collections.sort(terms);
+        assertThat(terms.size(), equalTo(2));
+        assertThat(terms.get(0).field(), equalTo("_field1"));
+        assertThat(terms.get(0).text(), equalTo("_term1"));
+        assertThat(terms.get(1).field(), equalTo("_field2"));
+        assertThat(terms.get(1).text(), equalTo("_term2"));
+    }
+
     public void testExtractQueryMetadata_phraseQuery() {
         PhraseQuery phraseQuery = new PhraseQuery("_field", "_term1", "term2");
         List<Term> terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(phraseQuery));
@@ -196,6 +217,31 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         assertThat(terms.get(0).bytes(), equalTo(termQuery1.getTerm().bytes()));
     }
 
+    public void testExtractQueryMetadata_commonTermsQuery() {
+        CommonTermsQuery commonTermsQuery = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 100);
+        commonTermsQuery.add(new Term("_field", "_term1"));
+        commonTermsQuery.add(new Term("_field", "_term2"));
+        List<Term> terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(commonTermsQuery));
+        Collections.sort(terms);
+        assertThat(terms.size(), equalTo(2));
+        assertThat(terms.get(0).field(), equalTo("_field"));
+        assertThat(terms.get(0).text(), equalTo("_term1"));
+        assertThat(terms.get(1).field(), equalTo("_field"));
+        assertThat(terms.get(1).text(), equalTo("_term2"));
+    }
+
+    public void testExtractQueryMetadata_blendedTermQuery() {
+        Term[] terms = new Term[]{new Term("_field", "_term1"), new Term("_field", "_term2")};
+        BlendedTermQuery commonTermsQuery = BlendedTermQuery.booleanBlendedQuery(terms, false);
+        List<Term> result = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(commonTermsQuery));
+        Collections.sort(result);
+        assertThat(result.size(), equalTo(2));
+        assertThat(result.get(0).field(), equalTo("_field"));
+        assertThat(result.get(0).text(), equalTo("_term1"));
+        assertThat(result.get(1).field(), equalTo("_field"));
+        assertThat(result.get(1).text(), equalTo("_term2"));
+    }
+
     public void testExtractQueryMetadata_unsupportedQuery() {
         TermRangeQuery termRangeQuery = new TermRangeQuery("_field", null, null, true, false);
 
@@ -207,7 +253,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         }
 
         TermQuery termQuery1 = new TermQuery(new Term("_field", "_term"));
-        BooleanQuery.Builder builder = new BooleanQuery.Builder();;
+        BooleanQuery.Builder builder = new BooleanQuery.Builder();
         builder.add(termQuery1, BooleanClause.Occur.SHOULD);
         builder.add(termRangeQuery, BooleanClause.Occur.SHOULD);
         BooleanQuery bq = builder.build();
diff --git a/core/src/test/java/org/elasticsearch/index/percolator/PercolatorFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/percolator/PercolatorFieldMapperTests.java
index 074e64f8232..c803cc9624a 100644
--- a/core/src/test/java/org/elasticsearch/index/percolator/PercolatorFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/percolator/PercolatorFieldMapperTests.java
@@ -26,13 +26,14 @@ import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.percolator.PercolatorService;
+import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.junit.Before;
 
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 
 public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
 
@@ -48,28 +49,40 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
             .endObject().endObject().string();
         mapperService.merge("type", new CompressedXContent(mapper), MapperService.MergeReason.MAPPING_UPDATE, true);
 
-        String percolatorMapper = XContentFactory.jsonBuilder().startObject().startObject(PercolatorService.TYPE_NAME)
+        String percolatorMapper = XContentFactory.jsonBuilder().startObject().startObject(PercolatorFieldMapper.TYPE_NAME)
             .startObject("properties").startObject("query").field("type", "percolator").endObject().endObject()
             .endObject().endObject().string();
-        mapperService.merge(PercolatorService.TYPE_NAME, new CompressedXContent(percolatorMapper), MapperService.MergeReason.MAPPING_UPDATE, true);
+        mapperService.merge(PercolatorFieldMapper.TYPE_NAME, new CompressedXContent(percolatorMapper), MapperService.MergeReason.MAPPING_UPDATE, true);
     }
 
     public void testPercolatorFieldMapper() throws Exception {
-        ParsedDocument doc = mapperService.documentMapper(PercolatorService.TYPE_NAME).parse("test", PercolatorService.TYPE_NAME, "1", XContentFactory.jsonBuilder().startObject()
+        ParsedDocument doc = mapperService.documentMapper(PercolatorFieldMapper.TYPE_NAME).parse("test", PercolatorFieldMapper.TYPE_NAME, "1", XContentFactory.jsonBuilder().startObject()
             .field("query", termQuery("field", "value"))
             .endObject().bytes());
 
         assertThat(doc.rootDoc().getFields(PercolatorFieldMapper.EXTRACTED_TERMS_FULL_FIELD_NAME).length, equalTo(1));
         assertThat(doc.rootDoc().getFields(PercolatorFieldMapper.EXTRACTED_TERMS_FULL_FIELD_NAME)[0].binaryValue().utf8ToString(), equalTo("field\0value"));
+        assertThat(doc.rootDoc().getFields(PercolatorFieldMapper.QUERY_BUILDER_FULL_FIELD_NAME).length, equalTo(1));
     }
 
+    public void testPercolatorFieldMapperUnMappedField() throws Exception {
+        MapperParsingException exception = expectThrows(MapperParsingException.class, () -> {
+            mapperService.documentMapper(PercolatorFieldMapper.TYPE_NAME).parse("test", PercolatorFieldMapper.TYPE_NAME, "1", XContentFactory.jsonBuilder().startObject()
+                    .field("query", termQuery("unmapped_field", "value"))
+                    .endObject().bytes());
+        });
+        assertThat(exception.getCause(), instanceOf(QueryShardException.class));
+        assertThat(exception.getCause().getMessage(), equalTo("No field mapping can be found for the field with name [unmapped_field]"));
+    }
+
+
     public void testPercolatorFieldMapper_noQuery() throws Exception {
-        ParsedDocument doc = mapperService.documentMapper(PercolatorService.TYPE_NAME).parse("test", PercolatorService.TYPE_NAME, "1", XContentFactory.jsonBuilder().startObject()
+        ParsedDocument doc = mapperService.documentMapper(PercolatorFieldMapper.TYPE_NAME).parse("test", PercolatorFieldMapper.TYPE_NAME, "1", XContentFactory.jsonBuilder().startObject()
             .endObject().bytes());
         assertThat(doc.rootDoc().getFields(PercolatorFieldMapper.EXTRACTED_TERMS_FULL_FIELD_NAME).length, equalTo(0));
 
         try {
-            mapperService.documentMapper(PercolatorService.TYPE_NAME).parse("test", PercolatorService.TYPE_NAME, "1", XContentFactory.jsonBuilder().startObject()
+            mapperService.documentMapper(PercolatorFieldMapper.TYPE_NAME).parse("test", PercolatorFieldMapper.TYPE_NAME, "1", XContentFactory.jsonBuilder().startObject()
                 .nullField("query")
                 .endObject().bytes());
         } catch (MapperParsingException e) {
@@ -81,11 +94,11 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
         IndexService indexService = createIndex("test1", Settings.EMPTY);
         MapperService mapperService = indexService.mapperService();
 
-        String percolatorMapper = XContentFactory.jsonBuilder().startObject().startObject(PercolatorService.TYPE_NAME)
+        String percolatorMapper = XContentFactory.jsonBuilder().startObject().startObject(PercolatorFieldMapper.TYPE_NAME)
             .startObject("properties").startObject("query").field("type", "percolator").field("index", "no").endObject().endObject()
             .endObject().endObject().string();
         try {
-            mapperService.merge(PercolatorService.TYPE_NAME, new CompressedXContent(percolatorMapper), MapperService.MergeReason.MAPPING_UPDATE, true);
+            mapperService.merge(PercolatorFieldMapper.TYPE_NAME, new CompressedXContent(percolatorMapper), MapperService.MergeReason.MAPPING_UPDATE, true);
             fail("MapperParsingException expected");
         } catch (MapperParsingException e) {
             assertThat(e.getMessage(), equalTo("Mapping definition for [query] has unsupported parameters:  [index : no]"));
diff --git a/core/src/test/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhaseTests.java b/core/src/test/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhaseTests.java
new file mode 100644
index 00000000000..5e37f1ce6e8
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhaseTests.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.percolator;
+
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.index.query.PercolatorQuery;
+import org.elasticsearch.search.highlight.SearchContextHighlight;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.test.ESTestCase;
+import org.mockito.Mockito;
+
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.sameInstance;
+
+public class PercolatorHighlightSubFetchPhaseTests extends ESTestCase {
+
+    public void testHitsExecutionNeeded() {
+        PercolatorQuery percolatorQuery = new PercolatorQuery.Builder("", ctx -> null, new BytesArray("{}"),
+                Mockito.mock(IndexSearcher.class), new MatchAllDocsQuery())
+                .build();
+
+        PercolatorHighlightSubFetchPhase subFetchPhase = new PercolatorHighlightSubFetchPhase(null);
+        SearchContext searchContext = Mockito.mock(SearchContext.class);
+        Mockito.when(searchContext.highlight()).thenReturn(new SearchContextHighlight(Collections.emptyList()));
+        Mockito.when(searchContext.query()).thenReturn(new MatchAllDocsQuery());
+
+        assertThat(subFetchPhase.hitsExecutionNeeded(searchContext), is(false));
+        IllegalStateException exception = expectThrows(IllegalStateException.class,
+                () -> subFetchPhase.hitsExecute(searchContext, null));
+        assertThat(exception.getMessage(), equalTo("couldn't locate percolator query"));
+
+        Mockito.when(searchContext.query()).thenReturn(percolatorQuery);
+        assertThat(subFetchPhase.hitsExecutionNeeded(searchContext), is(true));
+    }
+
+    public void testLocatePercolatorQuery() {
+        PercolatorQuery percolatorQuery = new PercolatorQuery.Builder("", ctx -> null, new BytesArray("{}"),
+                Mockito.mock(IndexSearcher.class), new MatchAllDocsQuery())
+                .build();
+
+        assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(new MatchAllDocsQuery()), nullValue());
+        BooleanQuery.Builder bq = new BooleanQuery.Builder();
+        bq.add(new MatchAllDocsQuery(), BooleanClause.Occur.FILTER);
+        assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(bq.build()), nullValue());
+        bq.add(percolatorQuery, BooleanClause.Occur.FILTER);
+        assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(bq.build()), sameInstance(percolatorQuery));
+
+        ConstantScoreQuery constantScoreQuery = new ConstantScoreQuery(new MatchAllDocsQuery());
+        assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(constantScoreQuery), nullValue());
+        constantScoreQuery = new ConstantScoreQuery(percolatorQuery);
+        assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(constantScoreQuery), sameInstance(percolatorQuery));
+
+        BoostQuery boostQuery = new BoostQuery(new MatchAllDocsQuery(), 1f);
+        assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(boostQuery), nullValue());
+        boostQuery = new BoostQuery(percolatorQuery, 1f);
+        assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(boostQuery), sameInstance(percolatorQuery));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/index/percolator/PercolatorQueryCacheTests.java b/core/src/test/java/org/elasticsearch/index/percolator/PercolatorQueryCacheTests.java
new file mode 100644
index 00000000000..3f3d90bafbd
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/percolator/PercolatorQueryCacheTests.java
@@ -0,0 +1,352 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.percolator;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TieredMergePolicy;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexWarmer;
+import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
+import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
+import org.elasticsearch.index.query.BoolQueryParser;
+import org.elasticsearch.index.query.PercolatorQuery;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParser;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.TermQueryParser;
+import org.elasticsearch.index.query.WildcardQueryParser;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.similarity.SimilarityService;
+import org.elasticsearch.index.warmer.ShardIndexWarmerService;
+import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.indices.mapper.MapperRegistry;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class PercolatorQueryCacheTests extends ESTestCase {
+
+    private QueryShardContext queryShardContext;
+    private PercolatorQueryCache cache;
+
+    void initialize(Object... fields) throws IOException {
+        Settings settings = Settings.settingsBuilder()
+                .put("node.name", PercolatorQueryCacheTests.class.toString())
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+                .build();
+
+        Map<String, QueryParser<?>> queryParsers = new HashMap<>();
+        queryParsers.put("term", new TermQueryParser());
+        queryParsers.put("wildcard", new WildcardQueryParser());
+        queryParsers.put("bool", new BoolQueryParser());
+        IndicesQueriesRegistry indicesQueriesRegistry = new IndicesQueriesRegistry(settings, queryParsers);
+
+        Settings indexSettings = Settings.settingsBuilder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(new Index("_index", ClusterState.UNKNOWN_UUID), indexSettings);
+        SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
+        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
+        MapperRegistry mapperRegistry = new IndicesModule().getMapperRegistry();
+        MapperService mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry,
+                () -> queryShardContext);
+        mapperService.merge("type", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("type", fields).string()),
+                MapperService.MergeReason.MAPPING_UPDATE, false);
+        cache = new PercolatorQueryCache(idxSettings, () -> queryShardContext);
+        queryShardContext = new QueryShardContext(idxSettings, null, null, mapperService, similarityService, null,
+                    indicesQueriesRegistry, cache);
+    }
+
+    public void testLoadQueries() throws Exception {
+        Directory directory = newDirectory();
+        IndexWriter indexWriter = new IndexWriter(
+                directory,
+                newIndexWriterConfig(new MockAnalyzer(random()))
+        );
+
+        boolean legacyFormat = randomBoolean();
+        Version version = legacyFormat ? Version.V_2_0_0 : Version.CURRENT;
+
+        storeQuery("0", indexWriter, termQuery("field1", "value1"), true, legacyFormat);
+        storeQuery("1", indexWriter, wildcardQuery("field1", "v*"), true, legacyFormat);
+        storeQuery("2", indexWriter, boolQuery().must(termQuery("field1", "value1")).must(termQuery("field2", "value2")),
+                true, legacyFormat);
+        // dymmy docs should be skipped during loading:
+        Document doc = new Document();
+        doc.add(new StringField("dummy", "value", Field.Store.YES));
+        indexWriter.addDocument(doc);
+        storeQuery("4", indexWriter, termQuery("field2", "value2"), true, legacyFormat);
+        // only documents that .percolator type should be loaded:
+        storeQuery("5", indexWriter, termQuery("field2", "value2"), false, legacyFormat);
+        storeQuery("6", indexWriter, termQuery("field3", "value3"), true, legacyFormat);
+        indexWriter.forceMerge(1);
+
+        // also include queries for percolator docs marked as deleted:
+        indexWriter.deleteDocuments(new Term("id", "6"));
+        indexWriter.close();
+
+        ShardId shardId = new ShardId("_index", ClusterState.UNKNOWN_UUID, 0);
+        IndexReader indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(directory), shardId);
+        assertThat(indexReader.leaves().size(), equalTo(1));
+        assertThat(indexReader.numDeletedDocs(), equalTo(1));
+        assertThat(indexReader.maxDoc(), equalTo(7));
+
+        initialize("field1", "type=keyword", "field2", "type=keyword", "field3", "type=keyword");
+
+        PercolatorQueryCache.QueriesLeaf leaf = cache.loadQueries(indexReader.leaves().get(0), version);
+        assertThat(leaf.queries.size(), equalTo(5));
+        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("field1", "value1"))));
+        assertThat(leaf.getQuery(1), equalTo(new WildcardQuery(new Term("field1", "v*"))));
+        assertThat(leaf.getQuery(2), equalTo(new BooleanQuery.Builder()
+                .add(new TermQuery(new Term("field1", "value1")), BooleanClause.Occur.MUST)
+                .add(new TermQuery(new Term("field2", "value2")), BooleanClause.Occur.MUST)
+                .build()
+        ));
+        assertThat(leaf.getQuery(4), equalTo(new TermQuery(new Term("field2", "value2"))));
+        assertThat(leaf.getQuery(6), equalTo(new TermQuery(new Term("field3", "value3"))));
+
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testGetQueries() throws Exception {
+        Directory directory = newDirectory();
+        IndexWriter indexWriter = new IndexWriter(
+                directory,
+                newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE)
+        );
+
+        storeQuery("0", indexWriter, termQuery("a", "0"), true, false);
+        storeQuery("1", indexWriter, termQuery("a", "1"), true, false);
+        storeQuery("2", indexWriter, termQuery("a", "2"), true, false);
+        indexWriter.flush();
+        storeQuery("3", indexWriter, termQuery("a", "3"), true, false);
+        storeQuery("4", indexWriter, termQuery("a", "4"), true, false);
+        storeQuery("5", indexWriter, termQuery("a", "5"), true, false);
+        indexWriter.flush();
+        storeQuery("6", indexWriter, termQuery("a", "6"), true, false);
+        storeQuery("7", indexWriter, termQuery("a", "7"), true, false);
+        storeQuery("8", indexWriter, termQuery("a", "8"), true, false);
+        indexWriter.flush();
+        indexWriter.close();
+
+        ShardId shardId = new ShardId("_index", ClusterState.UNKNOWN_UUID , 0);
+        IndexReader indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(directory), shardId);
+        assertThat(indexReader.leaves().size(), equalTo(3));
+        assertThat(indexReader.maxDoc(), equalTo(9));
+
+        initialize("a", "type=keyword");
+
+        try {
+            cache.getQueries(indexReader.leaves().get(0));
+            fail("IllegalStateException expected");
+        } catch (IllegalStateException e) {
+            assertThat(e.getMessage(), equalTo("queries not loaded, queries should be have been preloaded during index warming..."));
+        }
+
+        IndexShard indexShard = mockIndexShard();
+        ThreadPool threadPool = mockThreadPool();
+        IndexWarmer.Listener listener = cache.createListener(threadPool);
+        listener.warmReader(indexShard, new Engine.Searcher("test", new IndexSearcher(indexReader)));
+        PercolatorQueryCacheStats stats = cache.getStats(shardId);
+        assertThat(stats.getNumQueries(), equalTo(9L));
+
+        PercolatorQuery.QueryRegistry.Leaf leaf = cache.getQueries(indexReader.leaves().get(0));
+        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "0"))));
+        assertThat(leaf.getQuery(1), equalTo(new TermQuery(new Term("a", "1"))));
+        assertThat(leaf.getQuery(2), equalTo(new TermQuery(new Term("a", "2"))));
+
+        leaf = cache.getQueries(indexReader.leaves().get(1));
+        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "3"))));
+        assertThat(leaf.getQuery(1), equalTo(new TermQuery(new Term("a", "4"))));
+        assertThat(leaf.getQuery(2), equalTo(new TermQuery(new Term("a", "5"))));
+
+        leaf = cache.getQueries(indexReader.leaves().get(2));
+        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "6"))));
+        assertThat(leaf.getQuery(1), equalTo(new TermQuery(new Term("a", "7"))));
+        assertThat(leaf.getQuery(2), equalTo(new TermQuery(new Term("a", "8"))));
+
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testInvalidateEntries() throws Exception {
+        Directory directory = newDirectory();
+        IndexWriter indexWriter = new IndexWriter(
+                directory,
+                newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE)
+        );
+
+        storeQuery("0", indexWriter, termQuery("a", "0"), true, false);
+        indexWriter.flush();
+        storeQuery("1", indexWriter, termQuery("a", "1"), true, false);
+        indexWriter.flush();
+        storeQuery("2", indexWriter, termQuery("a", "2"), true, false);
+        indexWriter.flush();
+
+        ShardId shardId = new ShardId("_index", ClusterState.UNKNOWN_UUID, 0);
+        IndexReader indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(indexWriter), shardId);
+        assertThat(indexReader.leaves().size(), equalTo(3));
+        assertThat(indexReader.maxDoc(), equalTo(3));
+
+        initialize("a", "type=keyword");
+
+        IndexShard indexShard = mockIndexShard();
+        ThreadPool threadPool = mockThreadPool();
+        IndexWarmer.Listener listener = cache.createListener(threadPool);
+        listener.warmReader(indexShard, new Engine.Searcher("test", new IndexSearcher(indexReader)));
+        assertThat(cache.getStats(shardId).getNumQueries(), equalTo(3L));
+
+        PercolatorQuery.QueryRegistry.Leaf leaf = cache.getQueries(indexReader.leaves().get(0));
+        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "0"))));
+
+        leaf = cache.getQueries(indexReader.leaves().get(1));
+        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "1"))));
+
+        leaf = cache.getQueries(indexReader.leaves().get(2));
+        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "2"))));
+
+        // change merge policy, so that merges will actually happen:
+        indexWriter.getConfig().setMergePolicy(new TieredMergePolicy());
+        indexWriter.deleteDocuments(new Term("id", "1"));
+        indexWriter.forceMergeDeletes();
+        indexReader.close();
+        indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(indexWriter), shardId);
+        assertThat(indexReader.leaves().size(), equalTo(2));
+        assertThat(indexReader.maxDoc(), equalTo(2));
+        listener.warmReader(indexShard, new Engine.Searcher("test", new IndexSearcher(indexReader)));
+        assertThat(cache.getStats(shardId).getNumQueries(), equalTo(2L));
+
+        leaf = cache.getQueries(indexReader.leaves().get(0));
+        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "0"))));
+
+        leaf = cache.getQueries(indexReader.leaves().get(1));
+        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "2"))));
+
+        indexWriter.forceMerge(1);
+        indexReader.close();
+        indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(indexWriter), shardId);
+        assertThat(indexReader.leaves().size(), equalTo(1));
+        assertThat(indexReader.maxDoc(), equalTo(2));
+        listener.warmReader(indexShard, new Engine.Searcher("test", new IndexSearcher(indexReader)));
+        assertThat(cache.getStats(shardId).getNumQueries(), equalTo(2L));
+
+        leaf = cache.getQueries(indexReader.leaves().get(0));
+        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "0"))));
+        assertThat(leaf.getQuery(1), equalTo(new TermQuery(new Term("a", "2"))));
+
+        indexWriter.close();
+        indexReader.close();
+        directory.close();
+    }
+
+    void storeQuery(String id, IndexWriter indexWriter, QueryBuilder queryBuilder, boolean typeField, boolean legacy) throws IOException {
+        Document doc = new Document();
+        doc.add(new StringField("id", id, Field.Store.NO));
+        if (typeField) {
+            doc.add(new StringField(TypeFieldMapper.NAME, PercolatorFieldMapper.TYPE_NAME, Field.Store.NO));
+        }
+        if (legacy) {
+            BytesReference percolatorQuery = XContentFactory.jsonBuilder().startObject()
+                    .field("query", queryBuilder)
+                    .endObject().bytes();
+            doc.add(new StoredField(
+                    SourceFieldMapper.NAME,
+                    percolatorQuery.array(), percolatorQuery.arrayOffset(), percolatorQuery.length())
+            );
+        } else {
+            BytesRef queryBuilderAsBytes = new BytesRef(
+                    XContentFactory.contentBuilder(PercolatorQueryCache.QUERY_BUILDER_CONTENT_TYPE).value(queryBuilder).bytes().toBytes()
+            );
+            doc.add(new BinaryDocValuesField(PercolatorFieldMapper.QUERY_BUILDER_FULL_FIELD_NAME, queryBuilderAsBytes));
+        }
+        indexWriter.addDocument(doc);
+    }
+
+    IndexShard mockIndexShard() {
+        IndexShard indexShard = mock(IndexShard.class);
+        ShardIndexWarmerService shardIndexWarmerService = mock(ShardIndexWarmerService.class);
+        when(shardIndexWarmerService.logger()).thenReturn(logger);
+        when(indexShard.warmerService()).thenReturn(shardIndexWarmerService);
+        IndexSettings indexSettings = new IndexSettings(
+                IndexMetaData.builder("_index").settings(Settings.builder()
+                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                ).build(),
+                Settings.EMPTY
+        );
+        when(indexShard.indexSettings()).thenReturn(indexSettings);
+        return indexShard;
+    }
+
+    ThreadPool mockThreadPool() {
+        ThreadPool threadPool = mock(ThreadPool.class);
+        when(threadPool.executor(anyString())).thenReturn(Runnable::run);
+        return threadPool;
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
index 53594f43080..29026416102 100644
--- a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
@@ -22,8 +22,6 @@ package org.elasticsearch.index.query;
 import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
 import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.core.io.JsonStringEncoder;
-
-import org.apache.lucene.index.memory.MemoryIndex;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
@@ -38,10 +36,10 @@ import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
 import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -75,6 +73,7 @@ import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.percolator.PercolatorQueryCache;
 import org.elasticsearch.index.query.support.QueryParsers;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.similarity.SimilarityService;
@@ -100,7 +99,6 @@ import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.TestSearchContext;
 import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.test.cluster.TestClusterService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPoolModule;
 import org.joda.time.DateTime;
@@ -123,6 +121,8 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
@@ -140,11 +140,12 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     protected static final String DATE_FIELD_NAME = "mapped_date";
     protected static final String OBJECT_FIELD_NAME = "mapped_object";
     protected static final String GEO_POINT_FIELD_NAME = "mapped_geo_point";
+    protected static final String GEO_POINT_FIELD_MAPPING = "type=geo_point,lat_lon=true,geohash=true,geohash_prefix=true";
     protected static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape";
-    protected static final String[] MAPPED_FIELD_NAMES = new String[] { STRING_FIELD_NAME, INT_FIELD_NAME, DOUBLE_FIELD_NAME,
-            BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, OBJECT_FIELD_NAME, GEO_POINT_FIELD_NAME, GEO_SHAPE_FIELD_NAME };
-    protected static final String[] MAPPED_LEAF_FIELD_NAMES = new String[] { STRING_FIELD_NAME, INT_FIELD_NAME, DOUBLE_FIELD_NAME,
-            BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, GEO_POINT_FIELD_NAME };
+    protected static final String[] MAPPED_FIELD_NAMES = new String[]{STRING_FIELD_NAME, INT_FIELD_NAME, DOUBLE_FIELD_NAME,
+            BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, OBJECT_FIELD_NAME, GEO_POINT_FIELD_NAME, GEO_SHAPE_FIELD_NAME};
+    protected static final String[] MAPPED_LEAF_FIELD_NAMES = new String[]{STRING_FIELD_NAME, INT_FIELD_NAME, DOUBLE_FIELD_NAME,
+            BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, GEO_POINT_FIELD_NAME};
     private static final int NUMBER_OF_TESTQUERIES = 20;
 
     private static Injector injector;
@@ -193,11 +194,13 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                 .build();
         Settings indexSettings = Settings.settingsBuilder()
                 .put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
+        final ThreadPool threadPool = new ThreadPool(settings);
         index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings);
-        final TestClusterService clusterService = new TestClusterService();
-        clusterService.setState(new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder().put(
+        ClusterService clusterService = createClusterService(threadPool);
+        setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder().put(
                 new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
+
         SettingsModule settingsModule = new SettingsModule(settings);
         settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
         final Client proxy = (Client) Proxy.newProxyInstance(
@@ -209,10 +212,10 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             @Override
             protected void configure() {
                 Settings settings = Settings.builder()
-                    .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                    // no file watching, so we don't need a ResourceWatcherService
-                    .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
-                    .build();
+                        .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+                        // no file watching, so we don't need a ResourceWatcherService
+                        .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
+                        .build();
                 MockScriptEngine mockScriptEngine = new MockScriptEngine();
                 Multibinder<ScriptEngineService> multibinder = Multibinder.newSetBinder(binder(), ScriptEngineService.class);
                 multibinder.addBinding().toInstance(mockScriptEngine);
@@ -228,7 +231,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                 try {
                     ScriptService scriptService = new ScriptService(settings, new Environment(settings), engines, null, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
                     bind(ScriptService.class).toInstance(scriptService);
-                } catch(IOException e) {
+                } catch (IOException e) {
                     throw new IllegalStateException("error while binding ScriptService", e);
                 }
             }
@@ -237,7 +240,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         injector = new ModulesBuilder().add(
                 new EnvironmentModule(new Environment(settings)),
                 settingsModule,
-                new ThreadPoolModule(new ThreadPool(settings)),
+                new ThreadPoolModule(threadPool),
                 new IndicesModule() {
                     @Override
                     public void configure() {
@@ -252,6 +255,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                     protected void configureSearch() {
                         // Skip me
                     }
+
                     @Override
                     protected void configureSuggesters() {
                         // Skip me
@@ -286,26 +290,27 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
 
             }
         });
+        PercolatorQueryCache percolatorQueryCache = new PercolatorQueryCache(idxSettings, () -> queryShardContext);
         indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
-        queryShardContext = new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry);
+        queryShardContext = new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry, percolatorQueryCache);
         //create some random type with some default field, those types will stick around for all of the subclasses
         currentTypes = new String[randomIntBetween(0, 5)];
         for (int i = 0; i < currentTypes.length; i++) {
             String type = randomAsciiOfLengthBetween(1, 10);
             mapperService.merge(type, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(type,
                     STRING_FIELD_NAME, "type=text",
-                    STRING_FIELD_NAME_2, "type=text",
+                    STRING_FIELD_NAME_2, "type=keyword",
                     INT_FIELD_NAME, "type=integer",
                     DOUBLE_FIELD_NAME, "type=double",
                     BOOLEAN_FIELD_NAME, "type=boolean",
                     DATE_FIELD_NAME, "type=date",
                     OBJECT_FIELD_NAME, "type=object",
-                    GEO_POINT_FIELD_NAME, "type=geo_point,lat_lon=true,geohash=true,geohash_prefix=true",
+                    GEO_POINT_FIELD_NAME, GEO_POINT_FIELD_MAPPING,
                     GEO_SHAPE_FIELD_NAME, "type=geo_shape"
             ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
             // also add mappings for two inner field in the object field
-            mapperService.merge(type, new CompressedXContent("{\"properties\":{\""+OBJECT_FIELD_NAME+"\":{\"type\":\"object\","
-                    + "\"properties\":{\""+DATE_FIELD_NAME+"\":{\"type\":\"date\"},\""+INT_FIELD_NAME+"\":{\"type\":\"integer\"}}}}}"),
+            mapperService.merge(type, new CompressedXContent("{\"properties\":{\"" + OBJECT_FIELD_NAME + "\":{\"type\":\"object\","
+                            + "\"properties\":{\"" + DATE_FIELD_NAME + "\":{\"type\":\"date\"},\"" + INT_FIELD_NAME + "\":{\"type\":\"integer\"}}}}}"),
                     MapperService.MergeReason.MAPPING_UPDATE, false);
             currentTypes[i] = type;
         }
@@ -314,6 +319,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
 
     @AfterClass
     public static void afterClass() throws Exception {
+        injector.getInstance(ClusterService.class).close();
         terminate(injector.getInstance(ThreadPool.class));
         injector = null;
         index = null;
@@ -340,6 +346,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
 
     @After
     public void afterTest() {
+        queryShardContext.setFieldStatsProvider(null);
         clientInvocationHandler.delegate = null;
         SearchContext.removeCurrent();
     }
@@ -418,9 +425,9 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             // we'd like to see the offending field name here
             assertThat(e.getMessage(), containsString("bogusField"));
         }
-     }
+    }
 
-     /**
+    /**
      * Test that adding additional object into otherwise correct query string
      * should always trigger some kind of Parsing Exception.
      */
@@ -689,7 +696,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             return new QueryParser() {
                 @Override
                 public String[] names() {
-                    return new String[] {EmptyQueryBuilder.NAME};
+                    return new String[]{EmptyQueryBuilder.NAME};
                 }
 
                 @Override
@@ -713,7 +720,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
                 QueryBuilder<?> prototype = queryParser(query.getName()).getBuilderPrototype();
                 @SuppressWarnings("unchecked")
-                QB secondQuery = (QB)prototype.readFrom(in);
+                QB secondQuery = (QB) prototype.readFrom(in);
                 return secondQuery;
             }
         }
@@ -829,7 +836,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             }
         } else {
             if (randomBoolean()) {
-                types = new String[] { MetaData.ALL };
+                types = new String[]{MetaData.ALL};
             } else {
                 types = new String[0];
             }
@@ -889,6 +896,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
 
     private static class ClientInvocationHandler implements InvocationHandler {
         AbstractQueryTestCase<?> delegate;
+
         @Override
         public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
             if (method.equals(Client.class.getMethod("get", GetRequest.class))) {
@@ -899,12 +907,12 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                     }
                 };
             } else if (method.equals(Client.class.getMethod("multiTermVectors", MultiTermVectorsRequest.class))) {
-                    return new PlainActionFuture<MultiTermVectorsResponse>() {
-                        @Override
-                        public MultiTermVectorsResponse get() throws InterruptedException, ExecutionException {
-                            return delegate.executeMultiTermVectors((MultiTermVectorsRequest) args[0]);
-                        }
-                    };
+                return new PlainActionFuture<MultiTermVectorsResponse>() {
+                    @Override
+                    public MultiTermVectorsResponse get() throws InterruptedException, ExecutionException {
+                        return delegate.executeMultiTermVectors((MultiTermVectorsRequest) args[0]);
+                    }
+                };
             } else if (method.equals(Object.class.getMethod("toString"))) {
                 return "MockClient";
             }
@@ -949,8 +957,8 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         source.toXContent(builder, ToXContent.EMPTY_PARAMS);
         assertEquals(
                 msg(expected, builder.string()),
-                expected.replaceAll("\\s+",""),
-                builder.string().replaceAll("\\s+",""));
+                expected.replaceAll("\\s+", ""),
+                builder.string().replaceAll("\\s+", ""));
     }
 
     private static String msg(String left, String right) {
@@ -963,7 +971,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             } else {
                 builder.append(">> ").append("until offset: ").append(i)
                         .append(" [").append(left.charAt(i)).append(" vs.").append(right.charAt(i))
-                        .append("] [").append((int)left.charAt(i) ).append(" vs.").append((int)right.charAt(i)).append(']');
+                        .append("] [").append((int) left.charAt(i)).append(" vs.").append((int) right.charAt(i)).append(']');
                 return builder.toString();
             }
         }
@@ -972,7 +980,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             int rightEnd = Math.max(size, right.length()) - 1;
             builder.append(">> ").append("until offset: ").append(size)
                     .append(" [").append(left.charAt(leftEnd)).append(" vs.").append(right.charAt(rightEnd))
-                    .append("] [").append((int)left.charAt(leftEnd)).append(" vs.").append((int)right.charAt(rightEnd)).append(']');
+                    .append("] [").append((int) left.charAt(leftEnd)).append(" vs.").append((int) right.charAt(rightEnd)).append(']');
             return builder.toString();
         }
         return "";
diff --git a/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
index 832885b063a..5f26c0ce72e 100644
--- a/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
@@ -54,7 +54,7 @@ public class ConstantScoreQueryBuilderTests extends AbstractQueryTestCase<Consta
      * test that missing "filter" element causes {@link ParsingException}
      */
     public void testFilterElement() throws IOException {
-        String queryString = "{ \"" + ConstantScoreQueryBuilder.NAME + "\" : {}";
+        String queryString = "{ \"" + ConstantScoreQueryBuilder.NAME + "\" : {} }";
         try {
             parseQuery(queryString);
             fail("Expected ParsingException");
@@ -63,6 +63,38 @@ public class ConstantScoreQueryBuilderTests extends AbstractQueryTestCase<Consta
         }
     }
 
+    /**
+     * test that multiple "filter" elements causes {@link ParsingException}
+     */
+    public void testMultipleFilterElements() throws IOException {
+        String queryString = "{ \"" + ConstantScoreQueryBuilder.NAME + "\" : {\n" +
+                                    "\"filter\" : { \"term\": { \"foo\": \"a\" } },\n" +
+                                    "\"filter\" : { \"term\": { \"foo\": \"x\" } },\n" +
+                            "} }";
+        try {
+            parseQuery(queryString);
+            fail("Expected ParsingException");
+        } catch (ParsingException e) {
+            assertThat(e.getMessage(), containsString("accepts only one 'filter' element"));
+        }
+    }
+
+    /**
+     * test that "filter" does not accept an array of queries, throws {@link ParsingException}
+     */
+    public void testNoArrayAsFilterElements() throws IOException {
+        String queryString = "{ \"" + ConstantScoreQueryBuilder.NAME + "\" : {\n" +
+                                    "\"filter\" : [ { \"term\": { \"foo\": \"a\" } },\n" +
+                                                   "{ \"term\": { \"foo\": \"x\" } } ]\n" +
+                            "} }";
+        try {
+            parseQuery(queryString);
+            fail("Expected ParsingException");
+        } catch (ParsingException e) {
+            assertThat(e.getMessage(), containsString("unexpected token [START_ARRAY]"));
+        }
+    }
+
     public void testIllegalArguments() {
         try {
             new ConstantScoreQueryBuilder(null);
@@ -79,16 +111,16 @@ public class ConstantScoreQueryBuilderTests extends AbstractQueryTestCase<Consta
 
     public void testFromJson() throws IOException {
         String json =
-                "{\n" + 
-                "  \"constant_score\" : {\n" + 
-                "    \"filter\" : {\n" + 
-                "      \"terms\" : {\n" + 
-                "        \"user\" : [ \"kimchy\", \"elasticsearch\" ],\n" + 
-                "        \"boost\" : 42.0\n" + 
-                "      }\n" + 
-                "    },\n" + 
-                "    \"boost\" : 23.0\n" + 
-                "  }\n" + 
+                "{\n" +
+                "  \"constant_score\" : {\n" +
+                "    \"filter\" : {\n" +
+                "      \"terms\" : {\n" +
+                "        \"user\" : [ \"kimchy\", \"elasticsearch\" ],\n" +
+                "        \"boost\" : 42.0\n" +
+                "      }\n" +
+                "    },\n" +
+                "    \"boost\" : 23.0\n" +
+                "  }\n" +
                 "}";
 
         ConstantScoreQueryBuilder parsed = (ConstantScoreQueryBuilder) parseQuery(json);
diff --git a/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
index 0dc3a5ff90e..69fd843ad42 100644
--- a/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
@@ -24,7 +24,6 @@ import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.index.mapper.object.ObjectMapper;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -55,13 +54,8 @@ public class ExistsQueryBuilderTests extends AbstractQueryTestCase<ExistsQueryBu
     @Override
     protected void doAssertLuceneQuery(ExistsQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
         String fieldPattern = queryBuilder.fieldName();
-        ObjectMapper objectMapper = context.getObjectMapper(fieldPattern);
-        if (objectMapper != null) {
-            // automatic make the object mapper pattern
-            fieldPattern = fieldPattern + ".*";
-        }
         Collection<String> fields = context.simpleMatchToIndexNames(fieldPattern);
-        if (getCurrentTypes().length == 0 || fields.size() == 0) {
+        if (getCurrentTypes().length == 0) {
             assertThat(query, instanceOf(BooleanQuery.class));
             BooleanQuery booleanQuery = (BooleanQuery) query;
             assertThat(booleanQuery.clauses().size(), equalTo(0));
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java
index f07e695a1a0..cb0c374c5c0 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java
@@ -24,10 +24,12 @@ import org.apache.lucene.spatial.geopoint.search.GeoPointDistanceRangeQuery;
 import org.apache.lucene.spatial.util.GeoDistanceUtils;
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.Version;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.unit.DistanceUnit;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.search.geo.GeoDistanceRangeQuery;
 import org.elasticsearch.test.geo.RandomGeoGenerator;
 
@@ -296,6 +298,36 @@ public class GeoDistanceRangeQueryTests extends AbstractQueryTestCase<GeoDistanc
         }
     }
 
+    public void testNestedRangeQuery() throws IOException {
+        // create a nested geo_point type with a subfield named "geohash" (explicit testing for ISSUE #15179)
+        MapperService mapperService = queryShardContext().getMapperService();
+        String nestedMapping =
+            "{\"nested_doc\" : {\"properties\" : {" +
+            "\"locations\": {\"properties\": {" +
+            "\"geohash\": {\"type\": \"geo_point\"}}," +
+            "\"type\": \"nested\"}" +
+            "}}}";
+        mapperService.merge("nested_doc", new CompressedXContent(nestedMapping), MapperService.MergeReason.MAPPING_UPDATE, false);
+
+        // create a range query on the nested locations.geohash sub-field
+        String queryJson =
+            "{\n" +
+            "  \"nested\": {\n" +
+            "    \"path\": \"locations\",\n" +
+            "    \"query\": {\n" +
+            "      \"geo_distance_range\": {\n" +
+            "        \"from\": \"0.0km\",\n" +
+            "        \"to\" : \"200.0km\",\n" +
+            "        \"locations.geohash\": \"s7ws01wyd7ws\"\n" +
+            "      }\n" +
+            "    }\n" +
+            "  }\n" +
+            "}\n";
+        NestedQueryBuilder builder = (NestedQueryBuilder) parseQuery(queryJson);
+        QueryShardContext context = createShardContext();
+        builder.toQuery(context);
+    }
+
     public void testFromJson() throws IOException {
         String json =
                 "{\n" +
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeohashCellQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeohashCellQueryBuilderTests.java
index 7c94473e9dc..a05902aebf1 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeohashCellQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeohashCellQueryBuilderTests.java
@@ -63,7 +63,7 @@ public class GeohashCellQueryBuilderTests extends AbstractQueryTestCase<Builder>
             assertThat(query, instanceOf(TermQuery.class));
             TermQuery termQuery = (TermQuery) query;
             Term term = termQuery.getTerm();
-            assertThat(term.field(), equalTo(queryBuilder.fieldName() + GeoPointFieldMapper.Names.GEOHASH_SUFFIX));
+            assertThat(term.field(), equalTo(queryBuilder.fieldName() + "." + GeoPointFieldMapper.Names.GEOHASH));
             String geohash = queryBuilder.geohash();
             if (queryBuilder.precision() != null) {
                 int len = Math.min(queryBuilder.precision(), geohash.length());
diff --git a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
index 51c9d48218c..4715574b5a7 100644
--- a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
@@ -72,6 +72,7 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQue
         MapperService mapperService = queryShardContext().getMapperService();
         mapperService.merge(PARENT_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(PARENT_TYPE,
                 STRING_FIELD_NAME, "type=text",
+                STRING_FIELD_NAME_2, "type=keyword",
                 INT_FIELD_NAME, "type=integer",
                 DOUBLE_FIELD_NAME, "type=double",
                 BOOLEAN_FIELD_NAME, "type=boolean",
@@ -117,7 +118,7 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQue
     protected HasChildQueryBuilder doCreateTestQueryBuilder() {
         int min = randomIntBetween(0, Integer.MAX_VALUE / 2);
         int max = randomIntBetween(min, Integer.MAX_VALUE);
-        InnerHitsBuilder.InnerHit innerHit = new InnerHitsBuilder.InnerHit().setSize(100).addSort(STRING_FIELD_NAME, SortOrder.ASC);
+        InnerHitsBuilder.InnerHit innerHit = new InnerHitsBuilder.InnerHit().setSize(100).addSort(STRING_FIELD_NAME_2, SortOrder.ASC);
         return new HasChildQueryBuilder(CHILD_TYPE,
                 RandomQueryBuilder.createQuery(random()), max, min,
                 RandomPicks.randomFrom(random(), ScoreMode.values()),
@@ -145,7 +146,7 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQue
                 InnerHitsContext.BaseInnerHits innerHits = SearchContext.current().innerHits().getInnerHits().get("inner_hits_name");
                 assertEquals(innerHits.size(), 100);
                 assertEquals(innerHits.sort().getSort().length, 1);
-                assertEquals(innerHits.sort().getSort()[0].getField(), STRING_FIELD_NAME);
+                assertEquals(innerHits.sort().getSort()[0].getField(), STRING_FIELD_NAME_2);
             } else {
                 assertThat(SearchContext.current().innerHits().getInnerHits().size(), equalTo(0));
             }
diff --git a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryParserTests.java b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryParserTests.java
index 7be9a6c74ff..dc8ebc6c4ba 100644
--- a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryParserTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryParserTests.java
@@ -27,22 +27,27 @@ import static org.hamcrest.Matchers.is;
 public class HasChildQueryParserTests extends ESTestCase {
     public void testMinFromString() {
         assertThat("fromString(min) != MIN", ScoreMode.Min, equalTo(HasChildQueryParser.parseScoreMode("min")));
+        assertThat("min", equalTo(HasChildQueryParser.scoreModeAsString(ScoreMode.Min)));
     }
 
     public void testMaxFromString() {
         assertThat("fromString(max) != MAX", ScoreMode.Max, equalTo(HasChildQueryParser.parseScoreMode("max")));
+        assertThat("max", equalTo(HasChildQueryParser.scoreModeAsString(ScoreMode.Max)));
     }
 
     public void testAvgFromString() {
         assertThat("fromString(avg) != AVG", ScoreMode.Avg, equalTo(HasChildQueryParser.parseScoreMode("avg")));
+        assertThat("avg", equalTo(HasChildQueryParser.scoreModeAsString(ScoreMode.Avg)));
     }
 
     public void testSumFromString() {
-        assertThat("fromString(total) != SUM", ScoreMode.Total, equalTo(HasChildQueryParser.parseScoreMode("total")));
+        assertThat("fromString(total) != SUM", ScoreMode.Total, equalTo(HasChildQueryParser.parseScoreMode("sum")));
+        assertThat("sum", equalTo(HasChildQueryParser.scoreModeAsString(ScoreMode.Total)));
     }
 
     public void testNoneFromString() {
         assertThat("fromString(none) != NONE", ScoreMode.None, equalTo(HasChildQueryParser.parseScoreMode("none")));
+        assertThat("none", equalTo(HasChildQueryParser.scoreModeAsString(ScoreMode.None)));
     }
 
     /**
diff --git a/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
index b6a288d4590..e735d9a5481 100644
--- a/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
@@ -58,6 +58,7 @@ public class HasParentQueryBuilderTests extends AbstractQueryTestCase<HasParentQ
         MapperService mapperService = queryShardContext().getMapperService();
         mapperService.merge(PARENT_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(PARENT_TYPE,
                 STRING_FIELD_NAME, "type=text",
+                STRING_FIELD_NAME_2, "type=keyword",
                 INT_FIELD_NAME, "type=integer",
                 DOUBLE_FIELD_NAME, "type=double",
                 BOOLEAN_FIELD_NAME, "type=boolean",
@@ -67,6 +68,7 @@ public class HasParentQueryBuilderTests extends AbstractQueryTestCase<HasParentQ
         mapperService.merge(CHILD_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(CHILD_TYPE,
                 "_parent", "type=" + PARENT_TYPE,
                 STRING_FIELD_NAME, "type=text",
+                STRING_FIELD_NAME_2, "type=keyword",
                 INT_FIELD_NAME, "type=integer",
                 DOUBLE_FIELD_NAME, "type=double",
                 BOOLEAN_FIELD_NAME, "type=boolean",
@@ -102,7 +104,7 @@ public class HasParentQueryBuilderTests extends AbstractQueryTestCase<HasParentQ
      */
     @Override
     protected HasParentQueryBuilder doCreateTestQueryBuilder() {
-        InnerHitsBuilder.InnerHit innerHit = new InnerHitsBuilder.InnerHit().setSize(100).addSort(STRING_FIELD_NAME, SortOrder.ASC);
+        InnerHitsBuilder.InnerHit innerHit = new InnerHitsBuilder.InnerHit().setSize(100).addSort(STRING_FIELD_NAME_2, SortOrder.ASC);
         return new HasParentQueryBuilder(PARENT_TYPE,
                 RandomQueryBuilder.createQuery(random()),randomBoolean(),
                 randomBoolean() ? null : new QueryInnerHits("inner_hits_name", innerHit));
@@ -127,7 +129,7 @@ public class HasParentQueryBuilderTests extends AbstractQueryTestCase<HasParentQ
                 InnerHitsContext.BaseInnerHits innerHits = SearchContext.current().innerHits().getInnerHits().get("inner_hits_name");
                 assertEquals(innerHits.size(), 100);
                 assertEquals(innerHits.sort().getSort().length, 1);
-                assertEquals(innerHits.sort().getSort()[0].getField(), STRING_FIELD_NAME);
+                assertEquals(innerHits.sort().getSort()[0].getField(), STRING_FIELD_NAME_2);
             } else {
                 assertThat(SearchContext.current().innerHits().getInnerHits().size(), equalTo(0));
             }
diff --git a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
index 0f7e2e67e38..beef2df15d4 100644
--- a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
@@ -52,6 +52,7 @@ public class NestedQueryBuilderTests extends AbstractQueryTestCase<NestedQueryBu
                 BOOLEAN_FIELD_NAME, "type=boolean",
                 DATE_FIELD_NAME, "type=date",
                 OBJECT_FIELD_NAME, "type=object",
+                GEO_POINT_FIELD_NAME, GEO_POINT_FIELD_MAPPING,
                 "nested1", "type=nested"
         ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
     }
diff --git a/core/src/test/java/org/elasticsearch/index/query/ParentIdQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ParentIdQueryBuilderTests.java
index 8ae2145533e..aa38716d4e6 100644
--- a/core/src/test/java/org/elasticsearch/index/query/ParentIdQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/ParentIdQueryBuilderTests.java
@@ -19,12 +19,15 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.DocValuesTermsQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
 import org.elasticsearch.search.fetch.innerhits.InnerHitsContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.TestSearchContext;
@@ -87,11 +90,17 @@ public class ParentIdQueryBuilderTests extends AbstractQueryTestCase<ParentIdQue
 
     @Override
     protected void doAssertLuceneQuery(ParentIdQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        assertThat(query, Matchers.instanceOf(DocValuesTermsQuery.class));
-        DocValuesTermsQuery termsQuery = (DocValuesTermsQuery) query;
+        assertThat(query, Matchers.instanceOf(BooleanQuery.class));
+        BooleanQuery booleanQuery = (BooleanQuery) query;
+        assertThat(booleanQuery.clauses().size(), Matchers.equalTo(2));
+        DocValuesTermsQuery idQuery = (DocValuesTermsQuery) booleanQuery.clauses().get(0).getQuery();
         // there are no getters to get the field and terms on DocValuesTermsQuery, so lets validate by creating a
         // new query based on the builder:
-        assertThat(termsQuery, Matchers.equalTo(new DocValuesTermsQuery("_parent#" + PARENT_TYPE, queryBuilder.getId())));
+        assertThat(idQuery, Matchers.equalTo(new DocValuesTermsQuery("_parent#" + PARENT_TYPE, queryBuilder.getId())));
+
+        TermQuery typeQuery = (TermQuery) booleanQuery.clauses().get(1).getQuery();
+        assertThat(typeQuery.getTerm().field(), Matchers.equalTo(TypeFieldMapper.NAME));
+        assertThat(typeQuery.getTerm().text(), Matchers.equalTo(queryBuilder.getType()));
     }
 
     public void testFromJson() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/index/query/PercolatorQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/PercolatorQueryBuilderTests.java
new file mode 100644
index 00000000000..30d4ec908b8
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/query/PercolatorQueryBuilderTests.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.query;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.script.Script;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class PercolatorQueryBuilderTests extends AbstractQueryTestCase<PercolatorQueryBuilder> {
+
+    private String indexedDocumentIndex;
+    private String indexedDocumentType;
+    private String indexedDocumentId;
+    private String indexedDocumentRouting;
+    private String indexedDocumentPreference;
+    private Long indexedDocumentVersion;
+    private BytesReference documentSource;
+
+    boolean indexedDocumentExists = true;
+
+    @Override
+    protected PercolatorQueryBuilder doCreateTestQueryBuilder() {
+        return doCreateTestQueryBuilder(randomBoolean());
+    }
+
+    private PercolatorQueryBuilder doCreateTestQueryBuilder(boolean indexedDocument) {
+        String docType = randomAsciiOfLength(4);
+        documentSource = randomSource();
+        if (indexedDocument) {
+            indexedDocumentIndex = randomAsciiOfLength(4);
+            indexedDocumentType = randomAsciiOfLength(4);
+            indexedDocumentId = randomAsciiOfLength(4);
+            indexedDocumentRouting = randomAsciiOfLength(4);
+            indexedDocumentPreference = randomAsciiOfLength(4);
+            indexedDocumentVersion = (long) randomIntBetween(0, Integer.MAX_VALUE);
+            return new PercolatorQueryBuilder(docType, indexedDocumentIndex, indexedDocumentType, indexedDocumentId,
+                    indexedDocumentRouting, indexedDocumentPreference, indexedDocumentVersion);
+        } else {
+            return new PercolatorQueryBuilder(docType, documentSource);
+        }
+    }
+
+    @Override
+    protected GetResponse executeGet(GetRequest getRequest) {
+        assertThat(getRequest.index(), Matchers.equalTo(indexedDocumentIndex));
+        assertThat(getRequest.type(), Matchers.equalTo(indexedDocumentType));
+        assertThat(getRequest.id(), Matchers.equalTo(indexedDocumentId));
+        assertThat(getRequest.routing(), Matchers.equalTo(indexedDocumentRouting));
+        assertThat(getRequest.preference(), Matchers.equalTo(indexedDocumentPreference));
+        assertThat(getRequest.version(), Matchers.equalTo(indexedDocumentVersion));
+        if (indexedDocumentExists) {
+            return new GetResponse(
+                    new GetResult(indexedDocumentIndex, indexedDocumentType, indexedDocumentId, 0L, true, documentSource,
+                            Collections.emptyMap())
+            );
+        } else {
+            return new GetResponse(
+                    new GetResult(indexedDocumentIndex, indexedDocumentType, indexedDocumentId, -1, false, null, Collections.emptyMap())
+            );
+        }
+    }
+
+    @Override
+    protected void doAssertLuceneQuery(PercolatorQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
+        assertThat(query, Matchers.instanceOf(PercolatorQuery.class));
+        PercolatorQuery percolatorQuery = (PercolatorQuery) query;
+        assertThat(percolatorQuery.getDocumentType(), Matchers.equalTo(queryBuilder.getDocumentType()));
+        assertThat(percolatorQuery.getDocumentSource(), Matchers.equalTo(documentSource));
+    }
+
+    @Override
+    public void testMustRewrite() throws IOException {
+        PercolatorQueryBuilder pqb = doCreateTestQueryBuilder(true);
+        try {
+            pqb.toQuery(queryShardContext());
+            fail("IllegalStateException expected");
+        } catch (IllegalStateException e) {
+            assertThat(e.getMessage(), equalTo("query builder must be rewritten first"));
+        }
+        QueryBuilder<?> rewrite = pqb.rewrite(queryShardContext());
+        PercolatorQueryBuilder geoShapeQueryBuilder = new PercolatorQueryBuilder(pqb.getDocumentType(), documentSource);
+        assertEquals(geoShapeQueryBuilder, rewrite);
+    }
+
+    public void testIndexedDocumentDoesNotExist() throws IOException {
+        indexedDocumentExists = false;
+        PercolatorQueryBuilder pqb = doCreateTestQueryBuilder(true);
+        try {
+            pqb.rewrite(queryShardContext());
+            fail("ResourceNotFoundException expected");
+        } catch (ResourceNotFoundException e) {
+            String expectedString = "indexed document [" + indexedDocumentIndex + "/" + indexedDocumentType + "/" +
+                    indexedDocumentId +  "] couldn't be found";
+            assertThat(e.getMessage() , equalTo(expectedString));
+        }
+    }
+
+    // overwrite this test, because adding bogus field to the document part is valid and that would make the test fail
+    // (the document part represents the document being percolated and any key value pair is allowed there)
+    public void testUnknownObjectException() throws IOException {
+        String validQuery = createTestQueryBuilder().toString();
+        int endPos = validQuery.indexOf("document");
+        if (endPos == -1) {
+            endPos = validQuery.length();
+        }
+        assertThat(validQuery, containsString("{"));
+        for (int insertionPosition = 0; insertionPosition < endPos; insertionPosition++) {
+            if (validQuery.charAt(insertionPosition) == '{') {
+                String testQuery = validQuery.substring(0, insertionPosition) + "{ \"newField\" : " +
+                        validQuery.substring(insertionPosition) + "}";
+                try {
+                    parseQuery(testQuery);
+                    fail("some parsing exception expected for query: " + testQuery);
+                } catch (ParsingException | Script.ScriptParseException | ElasticsearchParseException e) {
+                    // different kinds of exception wordings depending on location
+                    // of mutation, so no simple asserts possible here
+                } catch (JsonParseException e) {
+                    // mutation produced invalid json
+                }
+            }
+        }
+    }
+
+    public void testRequiredParameters() {
+        try {
+            QueryBuilders.percolatorQuery(null, new BytesArray("{}"));
+            fail("IllegalArgumentException expected");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), equalTo("[document_type] is a required argument"));
+        }
+        try {
+            QueryBuilders.percolatorQuery("_document_type", null);
+            fail("IllegalArgumentException expected");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), equalTo("[document] is a required argument"));
+        }
+        try {
+            QueryBuilders.percolatorQuery(null, "_index", "_type", "_id", null, null, null);
+            fail("IllegalArgumentException expected");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), equalTo("[document_type] is a required argument"));
+        }
+        try {
+            QueryBuilders.percolatorQuery("_document_type", null, "_type", "_id", null, null, null);
+            fail("IllegalArgumentException expected");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), equalTo("[index] is a required argument"));
+        }
+        try {
+            QueryBuilders.percolatorQuery("_document_type", "_index", null, "_id", null, null, null);
+            fail("IllegalArgumentException expected");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), equalTo("[type] is a required argument"));
+        }
+        try {
+            QueryBuilders.percolatorQuery("_document_type", "_index", "_type", null, null, null, null);
+            fail("IllegalArgumentException expected");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), equalTo("[id] is a required argument"));
+        }
+    }
+
+    public void testFromJsonNoDocumentType() throws IOException {
+        try {
+            parseQuery("{\"percolator\" : { \"document\": {}}");
+            fail("IllegalArgumentException expected");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), equalTo("[percolator] query is missing required [document_type] parameter"));
+        }
+    }
+
+    private static BytesReference randomSource() {
+        try {
+            XContentBuilder xContent = XContentFactory.jsonBuilder();
+            xContent.map(RandomDocumentPicks.randomSource(random()));
+            return xContent.bytes();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolatorQueryTests.java b/core/src/test/java/org/elasticsearch/index/query/PercolatorQueryTests.java
similarity index 75%
rename from core/src/test/java/org/elasticsearch/percolator/PercolatorQueryTests.java
rename to core/src/test/java/org/elasticsearch/index/query/PercolatorQueryTests.java
index 8bef9138567..cbcfba55b97 100644
--- a/core/src/test/java/org/elasticsearch/percolator/PercolatorQueryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/PercolatorQueryTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.percolator;
+package org.elasticsearch.index.query;
 
 import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.apache.lucene.document.Field;
@@ -32,9 +32,10 @@ import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.memory.MemoryIndex;
+import org.apache.lucene.queries.BlendedTermQuery;
+import org.apache.lucene.queries.CommonTermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.PhraseQuery;
@@ -44,11 +45,12 @@ import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
 import org.elasticsearch.index.percolator.ExtractQueryTermsService;
+import org.elasticsearch.index.percolator.PercolatorFieldMapper;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
 import org.junit.Before;
@@ -58,8 +60,6 @@ import java.util.HashMap;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.not;
 
 public class PercolatorQueryTests extends ESTestCase {
 
@@ -75,13 +75,22 @@ public class PercolatorQueryTests extends ESTestCase {
 
     private Directory directory;
     private IndexWriter indexWriter;
-    private Map<BytesRef, Query> queries;
+    private Map<String, Query> queries;
+    private PercolatorQuery.QueryRegistry queryRegistry;
     private DirectoryReader directoryReader;
 
     @Before
     public void init() throws Exception {
         directory = newDirectory();
         queries = new HashMap<>();
+        queryRegistry = ctx -> docId -> {
+            try {
+                String val = ctx.reader().document(docId).get(UidFieldMapper.NAME);
+                return queries.get(Uid.createUid(val).id());
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        };
         IndexWriterConfig config = new IndexWriterConfig(new WhitespaceAnalyzer());
         config.setMergePolicy(NoMergePolicy.INSTANCE);
         indexWriter = new IndexWriter(directory, config);
@@ -127,8 +136,10 @@ public class PercolatorQueryTests extends ESTestCase {
         IndexSearcher percolateSearcher = memoryIndex.createSearcher();
 
         PercolatorQuery.Builder builder = new PercolatorQuery.Builder(
+                "docType",
+                queryRegistry,
+                new BytesArray("{}"),
                 percolateSearcher,
-                queries,
                 new MatchAllDocsQuery()
         );
         builder.extractQueryTermsQuery(EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME);
@@ -142,39 +153,6 @@ public class PercolatorQueryTests extends ESTestCase {
         assertThat(topDocs.scoreDocs[4].doc, equalTo(7));
     }
 
-    public void testWithScoring() throws Exception {
-        addPercolatorQuery("1", new TermQuery(new Term("field", "brown")), "field", "value1");
-
-        indexWriter.close();
-        directoryReader = DirectoryReader.open(directory);
-        // don't use newSearcher, which randomizes similarity. if it gets classic sim, the test eats it,
-        // as the score becomes 1 due to querynorm.
-        IndexSearcher shardSearcher = new IndexSearcher(directoryReader);
-
-        MemoryIndex memoryIndex = new MemoryIndex();
-        memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer());
-        IndexSearcher percolateSearcher = memoryIndex.createSearcher();
-
-        PercolatorQuery.Builder builder = new PercolatorQuery.Builder(
-                percolateSearcher,
-                queries,
-                new MatchAllDocsQuery()
-        );
-        builder.extractQueryTermsQuery(EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME);
-        builder.setPercolateQuery(new TermQuery(new Term("field", "value1")));
-
-        PercolatorQuery percolatorQuery = builder.build();
-        TopDocs topDocs = shardSearcher.search(percolatorQuery, 1);
-        assertThat(topDocs.totalHits, equalTo(1));
-        assertThat(topDocs.scoreDocs.length, equalTo(1));
-        assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
-        assertThat(topDocs.scoreDocs[0].score, not(1f));
-
-        Explanation explanation = shardSearcher.explain(percolatorQuery, 0);
-        assertThat(explanation.isMatch(), is(true));
-        assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[0].score));
-    }
-
     public void testDuel() throws Exception {
         int numQueries = scaledRandomIntBetween(32, 256);
         for (int i = 0; i < numQueries; i++) {
@@ -200,37 +178,36 @@ public class PercolatorQueryTests extends ESTestCase {
             MemoryIndex memoryIndex = new MemoryIndex();
             String id = Integer.toString(i);
             memoryIndex.addField("field", id, new WhitespaceAnalyzer());
-            IndexSearcher percolateSearcher = memoryIndex.createSearcher();
-
-            PercolatorQuery.Builder builder1 = new PercolatorQuery.Builder(
-                    percolateSearcher,
-                    queries,
-                    new MatchAllDocsQuery()
-            );
-            // enables the optimization that prevents queries from being evaluated that don't match
-            builder1.extractQueryTermsQuery(EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME);
-            TopDocs topDocs1 = shardSearcher.search(builder1.build(), 10);
-
-            PercolatorQuery.Builder builder2 = new PercolatorQuery.Builder(
-                    percolateSearcher,
-                    queries,
-                    new MatchAllDocsQuery()
-            );
-            TopDocs topDocs2 = shardSearcher.search(builder2.build(), 10);
-
-            assertThat(topDocs1.totalHits, equalTo(topDocs2.totalHits));
-            assertThat(topDocs1.scoreDocs.length, equalTo(topDocs2.scoreDocs.length));
-            for (int j = 0; j < topDocs1.scoreDocs.length; j++) {
-                assertThat(topDocs1.scoreDocs[j].doc, equalTo(topDocs2.scoreDocs[j].doc));
-            }
+            duelRun(memoryIndex, shardSearcher);
         }
     }
 
+    public void testDuelSpecificQueries() throws Exception {
+        CommonTermsQuery commonTermsQuery = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 128);
+        commonTermsQuery.add(new Term("field", "quick"));
+        commonTermsQuery.add(new Term("field", "brown"));
+        commonTermsQuery.add(new Term("field", "fox"));
+        addPercolatorQuery("_id1", commonTermsQuery);
+
+        BlendedTermQuery blendedTermQuery = BlendedTermQuery.booleanBlendedQuery(new Term[]{new Term("field", "quick"),
+                new Term("field", "brown"), new Term("field", "fox")}, false);
+        addPercolatorQuery("_id2", blendedTermQuery);
+
+        indexWriter.close();
+        directoryReader = DirectoryReader.open(directory);
+        IndexSearcher shardSearcher = newSearcher(directoryReader);
+
+        MemoryIndex memoryIndex = new MemoryIndex();
+        memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer());
+        duelRun(memoryIndex, shardSearcher);
+    }
+
     void addPercolatorQuery(String id, Query query, String... extraFields) throws IOException {
-        queries.put(new BytesRef(id), query);
+        queries.put(id, query);
         ParseContext.Document document = new ParseContext.Document();
-        ExtractQueryTermsService.extractQueryTerms(query, document, EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME, EXTRACTED_TERMS_FIELD_TYPE);
-        document.add(new StoredField(UidFieldMapper.NAME, Uid.createUid(PercolatorService.TYPE_NAME, id)));
+        ExtractQueryTermsService.extractQueryTerms(query, document, EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME,
+                EXTRACTED_TERMS_FIELD_TYPE);
+        document.add(new StoredField(UidFieldMapper.NAME, Uid.createUid(PercolatorFieldMapper.TYPE_NAME, id)));
         assert extraFields.length % 2 == 0;
         for (int i = 0; i < extraFields.length; i++) {
             document.add(new StringField(extraFields[i], extraFields[++i], Field.Store.NO));
@@ -238,6 +215,35 @@ public class PercolatorQueryTests extends ESTestCase {
         indexWriter.addDocument(document);
     }
 
+    private void duelRun(MemoryIndex memoryIndex, IndexSearcher shardSearcher) throws IOException {
+        IndexSearcher percolateSearcher = memoryIndex.createSearcher();
+        PercolatorQuery.Builder builder1 = new PercolatorQuery.Builder(
+                "docType",
+                queryRegistry,
+                new BytesArray("{}"),
+                percolateSearcher,
+                new MatchAllDocsQuery()
+        );
+        // enables the optimization that prevents queries from being evaluated that don't match
+        builder1.extractQueryTermsQuery(EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME);
+        TopDocs topDocs1 = shardSearcher.search(builder1.build(), 10);
+
+        PercolatorQuery.Builder builder2 = new PercolatorQuery.Builder(
+                "docType",
+                queryRegistry,
+                new BytesArray("{}"),
+                percolateSearcher,
+                new MatchAllDocsQuery()
+        );
+        TopDocs topDocs2 = shardSearcher.search(builder2.build(), 10);
+
+        assertThat(topDocs1.totalHits, equalTo(topDocs2.totalHits));
+        assertThat(topDocs1.scoreDocs.length, equalTo(topDocs2.scoreDocs.length));
+        for (int j = 0; j < topDocs1.scoreDocs.length; j++) {
+            assertThat(topDocs1.scoreDocs[j].doc, equalTo(topDocs2.scoreDocs[j].doc));
+        }
+    }
+
     private final static class CustomQuery extends Query {
 
         private final Term term;
diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
index 110c93d429c..f705db3a537 100644
--- a/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
@@ -47,7 +47,7 @@ public class QueryShardContextTests extends ESTestCase {
         MapperService mapperService = mock(MapperService.class);
         when(mapperService.getIndexSettings()).thenReturn(indexSettings);
         QueryShardContext context = new QueryShardContext(
-            indexSettings, null, null, mapperService, null, null, null
+            indexSettings, null, null, mapperService, null, null, null, null
         );
 
         context.setAllowUnmappedFields(false);
diff --git a/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
index 9f99b85a294..eb373539994 100644
--- a/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
@@ -22,11 +22,16 @@ package org.elasticsearch.index.query;
 import org.apache.lucene.search.LegacyNumericRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.joda.DateMathParser;
 import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.index.fieldstats.FieldStatsProvider;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
+import org.joda.time.chrono.ISOChronology;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -38,6 +43,7 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.sameInstance;
 
 public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuilder> {
 
@@ -392,4 +398,466 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
             assertThat(e.getMessage(), equalTo("Deprecated field [_name] used, replaced by [query name is not supported in short version of range query]"));
         }
     }
+
+    public void testRewriteLongToMatchAll() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        int queryFromValue = randomIntBetween(-1000000, 1000000);
+        int queryToValue = randomIntBetween(queryFromValue, 2000000);
+        long shardMinValue = randomIntBetween(queryFromValue, queryToValue);
+        long shardMaxValue = randomIntBetween((int) shardMinValue, queryToValue);
+        query.from((long) queryFromValue);
+        query.to((long) queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.WITHIN;
+            }
+
+            @SuppressWarnings("unchecked")
+            @Override
+            public <T extends Comparable<T>> FieldStats<T> get(String field) throws IOException {
+                assertThat(field, equalTo(fieldName));
+                return (FieldStats<T>) new FieldStats.Long(randomLong(), randomLong(), randomLong(), randomLong(), shardMinValue,
+                        shardMaxValue);
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(RangeQueryBuilder.class));
+        RangeQueryBuilder rewrittenRange = (RangeQueryBuilder) rewritten;
+        assertThat(rewrittenRange.fieldName(), equalTo(fieldName));
+        assertThat(rewrittenRange.from(), equalTo(null));
+        assertThat(rewrittenRange.to(), equalTo(null));
+    }
+
+    public void testRewriteLongToMatchNone() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        int queryFromValue = randomIntBetween(-1000000, 1000000);
+        int queryToValue = randomIntBetween(queryFromValue, 2000000);
+        query.from((long) queryFromValue);
+        query.to((long) queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.DISJOINT;
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
+    }
+
+    public void testRewriteLongToSame() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        int queryFromValue = randomIntBetween(-1000000, 1000000);
+        int queryToValue = randomIntBetween(queryFromValue, 2000000);
+        query.from((long) queryFromValue);
+        query.to((long) queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.INTERSECTS;
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, sameInstance(query));
+    }
+
+    public void testRewriteDoubleToMatchAll() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        double queryFromValue = randomDoubleBetween(-1000000.0, 1000000.0, true);
+        double queryToValue = randomDoubleBetween(queryFromValue, 2000000, true);
+        double shardMinValue = randomDoubleBetween(queryFromValue, queryToValue, true);
+        double shardMaxValue = randomDoubleBetween(shardMinValue, queryToValue, true);
+        query.from(queryFromValue);
+        query.to(queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.WITHIN;
+            }
+
+            @SuppressWarnings("unchecked")
+            @Override
+            public <T extends Comparable<T>> FieldStats<T> get(String field) throws IOException {
+                assertThat(field, equalTo(fieldName));
+                return (FieldStats<T>) new FieldStats.Double(randomLong(), randomLong(), randomLong(), randomLong(), shardMinValue,
+                        shardMaxValue);
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(RangeQueryBuilder.class));
+        RangeQueryBuilder rewrittenRange = (RangeQueryBuilder) rewritten;
+        assertThat(rewrittenRange.fieldName(), equalTo(fieldName));
+        assertThat(rewrittenRange.from(), equalTo(null));
+        assertThat(rewrittenRange.to(), equalTo(null));
+    }
+
+    public void testRewriteDoubleToMatchNone() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        double queryFromValue = randomDoubleBetween(-1000000, 1000000, true);
+        double queryToValue = randomDoubleBetween(queryFromValue, 2000000, true);
+        query.from(queryFromValue);
+        query.to(queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.DISJOINT;
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
+    }
+
+    public void testRewriteDoubleToSame() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        double queryFromValue = randomDoubleBetween(-1000000, 1000000, true);
+        double queryToValue = randomDoubleBetween(queryFromValue, 2000000, true);
+        query.from(queryFromValue);
+        query.to(queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.INTERSECTS;
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, sameInstance(query));
+    }
+
+    public void testRewriteFloatToMatchAll() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        float queryFromValue = (float) randomDoubleBetween(-1000000.0, 1000000.0, true);
+        float queryToValue = (float) randomDoubleBetween(queryFromValue, 2000000, true);
+        float shardMinValue = (float) randomDoubleBetween(queryFromValue, queryToValue, true);
+        float shardMaxValue = (float) randomDoubleBetween(shardMinValue, queryToValue, true);
+        query.from(queryFromValue);
+        query.to(queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.WITHIN;
+            }
+
+            @SuppressWarnings("unchecked")
+            @Override
+            public <T extends Comparable<T>> FieldStats<T> get(String field) throws IOException {
+                assertThat(field, equalTo(fieldName));
+                return (FieldStats<T>) new FieldStats.Float(randomLong(), randomLong(), randomLong(), randomLong(), shardMinValue,
+                        shardMaxValue);
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(RangeQueryBuilder.class));
+        RangeQueryBuilder rewrittenRange = (RangeQueryBuilder) rewritten;
+        assertThat(rewrittenRange.fieldName(), equalTo(fieldName));
+        assertThat(rewrittenRange.from(), equalTo(null));
+        assertThat(rewrittenRange.to(), equalTo(null));
+    }
+
+    public void testRewriteFloatToMatchNone() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        float queryFromValue = (float) randomDoubleBetween(-1000000, 1000000, true);
+        float queryToValue = (float) randomDoubleBetween(queryFromValue, 2000000, true);
+        query.from(queryFromValue);
+        query.to(queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.DISJOINT;
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
+    }
+
+    public void testRewriteFloatToSame() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        float queryFromValue = (float) randomDoubleBetween(-1000000, 1000000, true);
+        float queryToValue = (float) randomDoubleBetween(queryFromValue, 2000000, true);
+        query.from(queryFromValue);
+        query.to(queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.INTERSECTS;
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, sameInstance(query));
+    }
+
+    public void testRewriteTextToMatchAll() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        String queryFromValue = "damson";
+        String queryToValue = "plum";
+        String shardMinValue = "grape";
+        String shardMaxValue = "orange";
+        query.from(queryFromValue);
+        query.to(queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.WITHIN;
+            }
+
+            @SuppressWarnings("unchecked")
+            @Override
+            public <T extends Comparable<T>> FieldStats<T> get(String field) throws IOException {
+                assertThat(field, equalTo(fieldName));
+                return (FieldStats<T>) new FieldStats.Text(randomLong(), randomLong(), randomLong(), randomLong(),
+                        new BytesRef(shardMinValue), new BytesRef(shardMaxValue));
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(RangeQueryBuilder.class));
+        RangeQueryBuilder rewrittenRange = (RangeQueryBuilder) rewritten;
+        assertThat(rewrittenRange.fieldName(), equalTo(fieldName));
+        assertThat(rewrittenRange.from(), equalTo(null));
+        assertThat(rewrittenRange.to(), equalTo(null));
+    }
+
+    public void testRewriteTextToMatchNone() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        String queryFromValue = "damson";
+        String queryToValue = "plum";
+        query.from(queryFromValue);
+        query.to(queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.DISJOINT;
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
+    }
+
+    public void testRewriteTextToSame() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        String queryFromValue = "damson";
+        String queryToValue = "plum";
+        query.from(queryFromValue);
+        query.to(queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.INTERSECTS;
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, sameInstance(query));
+    }
+
+    public void testRewriteDateToMatchAll() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        DateTime queryFromValue = new DateTime(2015, 1, 1, 0, 0, 0, ISOChronology.getInstanceUTC());
+        DateTime queryToValue = new DateTime(2016, 1, 1, 0, 0, 0, ISOChronology.getInstanceUTC());
+        DateTime shardMinValue = new DateTime(2015, 3, 1, 0, 0, 0, ISOChronology.getInstanceUTC());
+        DateTime shardMaxValue = new DateTime(2015, 9, 1, 0, 0, 0, ISOChronology.getInstanceUTC());
+        query.from(queryFromValue);
+        query.to(queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.WITHIN;
+            }
+
+            @SuppressWarnings("unchecked")
+            @Override
+            public <T extends Comparable<T>> FieldStats<T> get(String field) throws IOException {
+                assertThat(field, equalTo(fieldName));
+                return (FieldStats<T>) new FieldStats.Date(randomLong(), randomLong(), randomLong(), randomLong(),
+                        shardMinValue.getMillis(), shardMaxValue.getMillis(), null);
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(RangeQueryBuilder.class));
+        RangeQueryBuilder rewrittenRange = (RangeQueryBuilder) rewritten;
+        assertThat(rewrittenRange.fieldName(), equalTo(fieldName));
+        assertThat(rewrittenRange.from(), equalTo(null));
+        assertThat(rewrittenRange.to(), equalTo(null));
+    }
+
+    public void testRewriteDateWithNowToMatchAll() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        String queryFromValue = "now-2d";
+        String queryToValue = "now";
+        DateTime shardMinValue = new DateTime().minusHours(12);
+        DateTime shardMaxValue = new DateTime().minusHours(24);
+        query.from(queryFromValue);
+        query.to(queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.WITHIN;
+            }
+
+            @SuppressWarnings("unchecked")
+            @Override
+            public <T extends Comparable<T>> FieldStats<T> get(String field) throws IOException {
+                assertThat(field, equalTo(fieldName));
+                return (FieldStats<T>) new FieldStats.Date(randomLong(), randomLong(), randomLong(), randomLong(),
+                        shardMinValue.getMillis(), shardMaxValue.getMillis(), null);
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(RangeQueryBuilder.class));
+        RangeQueryBuilder rewrittenRange = (RangeQueryBuilder) rewritten;
+        assertThat(rewrittenRange.fieldName(), equalTo(fieldName));
+        assertThat(rewrittenRange.from(), equalTo(null));
+        assertThat(rewrittenRange.to(), equalTo(null));
+    }
+
+    public void testRewriteDateToMatchNone() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        DateTime queryFromValue = new DateTime(2015, 1, 1, 0, 0, 0, ISOChronology.getInstanceUTC());
+        DateTime queryToValue = new DateTime(2016, 1, 1, 0, 0, 0, ISOChronology.getInstanceUTC());
+        query.from(queryFromValue);
+        query.to(queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.DISJOINT;
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
+    }
+
+    public void testRewriteDateWithNowToMatchNone() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        String queryFromValue = "now-2d";
+        String queryToValue = "now";
+        query.from(queryFromValue);
+        query.to(queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.DISJOINT;
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
+    }
+
+    public void testRewriteDateToSame() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        DateTime queryFromValue = new DateTime(2015, 1, 1, 0, 0, 0, ISOChronology.getInstanceUTC());
+        DateTime queryToValue = new DateTime(2016, 1, 1, 0, 0, 0, ISOChronology.getInstanceUTC());
+        query.from(queryFromValue);
+        query.to(queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.INTERSECTS;
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, sameInstance(query));
+    }
+
+    public void testRewriteDateWithNowToSame() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName);
+        String queryFromValue = "now-2d";
+        String queryToValue = "now";
+        query.from(queryFromValue);
+        query.to(queryToValue);
+        QueryShardContext queryShardContext = queryShardContext();
+        FieldStatsProvider fieldStatsProvider = new FieldStatsProvider(null, null) {
+
+            @Override
+            public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+                    DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+                return Relation.INTERSECTS;
+            }
+        };
+        queryShardContext.setFieldStatsProvider(fieldStatsProvider);
+        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, sameInstance(query));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java
index 4b6788d463f..ab9379f7f54 100644
--- a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java
@@ -54,7 +54,6 @@ import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.AtomicFieldData;
 import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
@@ -89,11 +88,6 @@ public class FunctionScoreTests extends ESTestCase {
             return "test";
         }
 
-        @Override
-        public FieldDataType getFieldDataType() {
-            throw new UnsupportedOperationException(UNSUPPORTED);
-        }
-
         @Override
         public AtomicFieldData load(LeafReaderContext context) {
             return new AtomicFieldData() {
@@ -174,11 +168,6 @@ public class FunctionScoreTests extends ESTestCase {
             return "test";
         }
 
-        @Override
-        public FieldDataType getFieldDataType() {
-            throw new UnsupportedOperationException(UNSUPPORTED);
-        }
-
         @Override
         public AtomicNumericFieldData load(LeafReaderContext context) {
             return new AtomicNumericFieldData() {
diff --git a/core/src/test/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java b/core/src/test/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java
index 1758d95a554..ec405bd8407 100644
--- a/core/src/test/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java
+++ b/core/src/test/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java
@@ -68,7 +68,7 @@ public class CustomQueryParserIT extends ESIntegTestCase {
 
     private static QueryShardContext queryShardContext() {
         IndicesService indicesService = internalCluster().getDataNodeInstance(IndicesService.class);
-        return indicesService.indexServiceSafe("index").newQueryShardContext();
+        return indicesService.indexServiceSafe(resolveIndex("index")).newQueryShardContext();
     }
 
     //see #11120
diff --git a/core/src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java b/core/src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java
index 847cd3536e7..2dc87594c9d 100644
--- a/core/src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java
@@ -32,7 +32,6 @@ import org.apache.lucene.search.join.ScoreMode;
 import org.apache.lucene.search.join.ToParentBlockJoinQuery;
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
@@ -49,8 +48,8 @@ import static org.hamcrest.Matchers.equalTo;
 public class DoubleNestedSortingTests extends AbstractNumberNestedSortingTestCase {
 
     @Override
-    protected FieldDataType getFieldDataType() {
-        return new FieldDataType("double");
+    protected String getFieldDataType() {
+        return "double";
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java b/core/src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java
index e63dde2d7fe..fcecee82ed8 100644
--- a/core/src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java
@@ -32,7 +32,6 @@ import org.apache.lucene.search.join.ScoreMode;
 import org.apache.lucene.search.join.ToParentBlockJoinQuery;
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
@@ -49,8 +48,8 @@ import static org.hamcrest.Matchers.equalTo;
 public class FloatNestedSortingTests extends DoubleNestedSortingTests {
 
     @Override
-    protected FieldDataType getFieldDataType() {
-        return new FieldDataType("float");
+    protected String getFieldDataType() {
+        return "float";
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java b/core/src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java
index da7aca4297f..39994b0c057 100644
--- a/core/src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java
@@ -20,7 +20,6 @@ package org.elasticsearch.index.search.nested;
 
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.IndexableField;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
@@ -32,8 +31,8 @@ import org.elasticsearch.search.MultiValueMode;
 public class LongNestedSortingTests extends AbstractNumberNestedSortingTestCase {
 
     @Override
-    protected FieldDataType getFieldDataType() {
-        return new FieldDataType("long");
+    protected String getFieldDataType() {
+        return "long";
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java b/core/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java
index d5e9ff85ba3..f3576c67f82 100644
--- a/core/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java
@@ -42,9 +42,7 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.TestUtil;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.fielddata.AbstractFieldDataTestCase;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
 import org.elasticsearch.index.fielddata.NoOrdinalsStringFieldDataTests;
@@ -64,8 +62,8 @@ import static org.hamcrest.Matchers.equalTo;
  */
 public class NestedSortingTests extends AbstractFieldDataTestCase {
     @Override
-    protected FieldDataType getFieldDataType() {
-        return new FieldDataType("string", Settings.builder().put("format", "paged_bytes"));
+    protected String getFieldDataType() {
+        return "string";
     }
 
     public void testDuel() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index e70ca9ec6de..1839df5dd30 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -37,13 +37,15 @@ import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.TransportIndexAction;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterInfoService;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.AllocationId;
@@ -53,6 +55,7 @@ import org.elasticsearch.cluster.routing.ShardRoutingHelper;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -60,7 +63,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -70,6 +72,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.ShardLock;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.NodeServicesProvider;
@@ -84,6 +87,7 @@ import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
+import org.elasticsearch.index.percolator.PercolatorQueryCache;
 import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
 import org.elasticsearch.index.store.Store;
@@ -118,6 +122,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED;
+import static org.elasticsearch.common.lucene.Lucene.cleanLuceneIndex;
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -126,6 +131,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
@@ -141,25 +147,25 @@ public class IndexShardTests extends ESSingleNodeTestCase {
 
     public void testWriteShardState() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            ShardId id = new ShardId("foo", "_na_", 1);
+            ShardId id = new ShardId("foo", "fooUUID", 1);
             long version = between(1, Integer.MAX_VALUE / 2);
             boolean primary = randomBoolean();
             AllocationId allocationId = randomBoolean() ? null : randomAllocationId();
-            ShardStateMetaData state1 = new ShardStateMetaData(version, primary, "foo", allocationId);
+            ShardStateMetaData state1 = new ShardStateMetaData(version, primary, "fooUUID", allocationId);
             write(state1, env.availableShardPaths(id));
             ShardStateMetaData shardStateMetaData = load(logger, env.availableShardPaths(id));
             assertEquals(shardStateMetaData, state1);
 
-            ShardStateMetaData state2 = new ShardStateMetaData(version, primary, "foo", allocationId);
+            ShardStateMetaData state2 = new ShardStateMetaData(version, primary, "fooUUID", allocationId);
             write(state2, env.availableShardPaths(id));
             shardStateMetaData = load(logger, env.availableShardPaths(id));
             assertEquals(shardStateMetaData, state1);
 
-            ShardStateMetaData state3 = new ShardStateMetaData(version + 1, primary, "foo", allocationId);
+            ShardStateMetaData state3 = new ShardStateMetaData(version + 1, primary, "fooUUID", allocationId);
             write(state3, env.availableShardPaths(id));
             shardStateMetaData = load(logger, env.availableShardPaths(id));
             assertEquals(shardStateMetaData, state3);
-            assertEquals("foo", state3.indexUUID);
+            assertEquals("fooUUID", state3.indexUUID);
         }
     }
 
@@ -167,7 +173,10 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         createIndex("test");
         ensureGreen();
         NodeEnvironment env = getInstanceFromNode(NodeEnvironment.class);
-        Path[] shardPaths = env.availableShardPaths(new ShardId("test", "_na_", 0));
+
+        ClusterService cs = getInstanceFromNode(ClusterService.class);
+        final Index index = cs.state().metaData().index("test").getIndex();
+        Path[] shardPaths = env.availableShardPaths(new ShardId(index, 0));
         logger.info("--> paths: [{}]", (Object)shardPaths);
         // Should not be able to acquire the lock because it's already open
         try {
@@ -179,7 +188,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         // Test without the regular shard lock to assume we can acquire it
         // (worst case, meaning that the shard lock could be acquired and
         // we're green to delete the shard's directory)
-        ShardLock sLock = new DummyShardLock(new ShardId("test", "_na_", 0));
+        ShardLock sLock = new DummyShardLock(new ShardId(index, 0));
         try {
             env.deleteShardDirectoryUnderLock(sLock, IndexSettingsModule.newIndexSettings("test", Settings.EMPTY));
             fail("should not have been able to delete the directory");
@@ -193,7 +202,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         NodeEnvironment env = getInstanceFromNode(NodeEnvironment.class);
-        IndexService test = indicesService.indexService("test");
+        IndexService test = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
         ShardStateMetaData shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId));
         assertEquals(getShardStateMetadata(shard), shardStateMetaData);
@@ -226,7 +235,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         NodeEnvironment env = getInstanceFromNode(NodeEnvironment.class);
-        IndexService test = indicesService.indexService("test");
+        IndexService test = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
         // fail shard
         shard.failShard("test shard fail", new CorruptIndexException("", ""));
@@ -281,7 +290,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)).get());
         ensureGreen("test");
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService indexService = indicesService.indexServiceSafe("test");
+        IndexService indexService = indicesService.indexServiceSafe(resolveIndex("test"));
         IndexShard indexShard = indexService.getShardOrNull(0);
         client().admin().indices().prepareDelete("test").get();
         assertThat(indexShard.getActiveOperationsCount(), equalTo(0));
@@ -292,39 +301,141 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             // expected
         }
         try {
-            indexShard.acquireReplicaOperationLock();
+            indexShard.acquireReplicaOperationLock(indexShard.getPrimaryTerm());
             fail("we should not be able to increment anymore");
         } catch (IndexShardClosedException e) {
             // expected
         }
     }
 
-    public void testIndexOperationsCounter() throws InterruptedException, ExecutionException, IOException {
+    public void testOperationLocksOnPrimaryShards() throws InterruptedException, ExecutionException, IOException {
         assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)).get());
         ensureGreen("test");
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService indexService = indicesService.indexServiceSafe("test");
+        IndexService indexService = indicesService.indexServiceSafe(resolveIndex("test"));
         IndexShard indexShard = indexService.getShardOrNull(0);
+        long primaryTerm = indexShard.getPrimaryTerm();
+
+        ShardRouting temp = indexShard.routingEntry();
+        final ShardRouting newPrimaryShardRouting;
+        if (randomBoolean()) {
+            // relocation target
+            newPrimaryShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(), "other node",
+                true, ShardRoutingState.INITIALIZING, AllocationId.newRelocation(temp.allocationId()));
+        } else if (randomBoolean()) {
+            // simulate promotion
+            ShardRouting newReplicaShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(), null,
+                false, ShardRoutingState.STARTED, temp.allocationId());
+            indexShard.updateRoutingEntry(newReplicaShardRouting, false);
+            primaryTerm = primaryTerm + 1;
+            indexShard.updatePrimaryTerm(primaryTerm);
+            newPrimaryShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(), null,
+                true, ShardRoutingState.STARTED, temp.allocationId());
+        } else {
+            newPrimaryShardRouting = temp;
+        }
+        indexShard.updateRoutingEntry(newPrimaryShardRouting, false);
+
         assertEquals(0, indexShard.getActiveOperationsCount());
+        if (newPrimaryShardRouting.isRelocationTarget() == false) {
+            try {
+                indexShard.acquireReplicaOperationLock(primaryTerm);
+                fail("shard shouldn't accept operations as replica");
+            } catch (IllegalStateException ignored) {
+
+            }
+        }
         Releasable operation1 = indexShard.acquirePrimaryOperationLock();
         assertEquals(1, indexShard.getActiveOperationsCount());
         Releasable operation2 = indexShard.acquirePrimaryOperationLock();
         assertEquals(2, indexShard.getActiveOperationsCount());
+
+        Releasables.close(operation1, operation2);
+        assertEquals(0, indexShard.getActiveOperationsCount());
+    }
+
+    public void testOperationLocksOnReplicaShards() throws InterruptedException, ExecutionException, IOException {
+        assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)).get());
+        ensureGreen("test");
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        IndexService indexService = indicesService.indexServiceSafe(resolveIndex("test"));
+        IndexShard indexShard = indexService.getShardOrNull(0);
+        long primaryTerm = indexShard.getPrimaryTerm();
+
+        // ugly hack to allow the shard to operated as a replica
+        final ShardRouting temp = indexShard.routingEntry();
+        final ShardRouting newShardRouting;
+        switch (randomInt(2)) {
+            case 0:
+                // started replica
+                newShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(), null,
+                    false, ShardRoutingState.STARTED, AllocationId.newRelocation(temp.allocationId()));
+
+                indexShard.updateRoutingEntry(newShardRouting, false);
+                break;
+            case 1:
+                // initializing replica / primary
+                final boolean relocating = randomBoolean();
+                newShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(),
+                    relocating ? "sourceNode" : null,
+                    relocating ? randomBoolean() : false,
+                    ShardRoutingState.INITIALIZING,
+                    relocating ? AllocationId.newRelocation(temp.allocationId()) : temp.allocationId());
+                indexShard.updateRoutingEntry(newShardRouting, false);
+                break;
+            case 2:
+                // relocation source
+                newShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(), "otherNode",
+                    false, ShardRoutingState.RELOCATING, AllocationId.newRelocation(temp.allocationId()));
+                indexShard.updateRoutingEntry(newShardRouting, false);
+                indexShard.relocated("test");
+                break;
+            default:
+                throw new UnsupportedOperationException("get your numbers straight");
+
+        }
+        logger.info("updated shard routing to {}", newShardRouting);
+
+        assertEquals(0, indexShard.getActiveOperationsCount());
+        if (newShardRouting.primary() == false) {
+            try {
+                indexShard.acquirePrimaryOperationLock();
+                fail("shard shouldn't accept primary ops");
+            } catch (IllegalStateException ignored) {
+
+            }
+        }
+
+        Releasable operation1 = indexShard.acquireReplicaOperationLock(primaryTerm);
+        assertEquals(1, indexShard.getActiveOperationsCount());
+        Releasable operation2 = indexShard.acquireReplicaOperationLock(primaryTerm);
+        assertEquals(2, indexShard.getActiveOperationsCount());
+
+        try {
+            indexShard.acquireReplicaOperationLock(primaryTerm - 1);
+            fail("you can not increment the operation counter with an older primary term");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), containsString("operation term"));
+            assertThat(e.getMessage(), containsString("too old"));
+        }
+
+        // but you can increment with a newer one..
+        indexShard.acquireReplicaOperationLock(primaryTerm + 1 + randomInt(20)).close();
         Releasables.close(operation1, operation2);
         assertEquals(0, indexShard.getActiveOperationsCount());
     }
 
     public void testMarkAsInactiveTriggersSyncedFlush() throws Exception {
         assertAcked(client().admin().indices().prepareCreate("test")
-                .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0));
+            .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0));
         client().prepareIndex("test", "test").setSource("{}").get();
         ensureGreen("test");
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        indicesService.indexService("test").getShardOrNull(0).checkIdle(0);
+        indicesService.indexService(resolveIndex("test")).getShardOrNull(0).checkIdle(0);
         assertBusy(() -> {
             IndexStats indexStats = client().admin().indices().prepareStats("test").clear().get().getIndex("test");
             assertNotNull(indexStats.getShards()[0].getCommitStats().getUserData().get(Engine.SYNC_COMMIT_ID));
-            indicesService.indexService("test").getShardOrNull(0).checkIdle(0);
+            indicesService.indexService(resolveIndex("test")).getShardOrNull(0).checkIdle(0);
             }
         );
         IndexStats indexStats = client().admin().indices().prepareStats("test").get().getIndex("test");
@@ -345,7 +456,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         ensureGreen();
         client().prepareIndex("test", "bar", "1").setSource("{}").get();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("test");
+        IndexService test = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
         setDurability(shard, Translog.Durability.REQUEST);
         assertFalse(shard.getEngine().getTranslog().syncNeeded());
@@ -361,14 +472,14 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertTrue(shard.getEngine().getTranslog().syncNeeded());
         setDurability(shard, Translog.Durability.REQUEST);
         assertNoFailures(client().prepareBulk()
-                .add(client().prepareIndex("test", "bar", "3").setSource("{}"))
-                .add(client().prepareDelete("test", "bar", "1")).get());
+            .add(client().prepareIndex("test", "bar", "3").setSource("{}"))
+            .add(client().prepareDelete("test", "bar", "1")).get());
         assertFalse(shard.getEngine().getTranslog().syncNeeded());
 
         setDurability(shard, Translog.Durability.ASYNC);
         assertNoFailures(client().prepareBulk()
-                .add(client().prepareIndex("test", "bar", "4").setSource("{}"))
-                .add(client().prepareDelete("test", "bar", "3")).get());
+            .add(client().prepareIndex("test", "bar", "4").setSource("{}"))
+            .add(client().prepareDelete("test", "bar", "3")).get());
         setDurability(shard, Translog.Durability.REQUEST);
         assertTrue(shard.getEngine().getTranslog().syncNeeded());
     }
@@ -381,11 +492,11 @@ public class IndexShardTests extends ESSingleNodeTestCase {
     public void testMinimumCompatVersion() {
         Version versionCreated = VersionUtils.randomVersion(random());
         assertAcked(client().admin().indices().prepareCreate("test")
-                .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0, SETTING_VERSION_CREATED, versionCreated.id));
+            .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0, SETTING_VERSION_CREATED, versionCreated.id));
         client().prepareIndex("test", "test").setSource("{}").get();
         ensureGreen("test");
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexShard test = indicesService.indexService("test").getShardOrNull(0);
+        IndexShard test = indicesService.indexService(resolveIndex("test")).getShardOrNull(0);
         assertEquals(versionCreated.luceneVersion, test.minimumCompatibleVersion());
         client().prepareIndex("test", "test").setSource("{}").get();
         assertEquals(versionCreated.luceneVersion, test.minimumCompatibleVersion());
@@ -395,8 +506,8 @@ public class IndexShardTests extends ESSingleNodeTestCase {
 
     public void testUpdatePriority() {
         assertAcked(client().admin().indices().prepareCreate("test")
-                .setSettings(IndexMetaData.SETTING_PRIORITY, 200));
-        IndexService indexService = getInstanceFromNode(IndicesService.class).indexService("test");
+            .setSettings(IndexMetaData.SETTING_PRIORITY, 200));
+        IndexService indexService = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
         assertEquals(200, indexService.getIndexSettings().getSettings().getAsInt(IndexMetaData.SETTING_PRIORITY, 0).intValue());
         client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_PRIORITY, 400).build()).get();
         assertEquals(400, indexService.getIndexSettings().getSettings().getAsInt(IndexMetaData.SETTING_PRIORITY, 0).intValue());
@@ -410,7 +521,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         SearchResponse response = client().prepareSearch("test").get();
         assertHitCount(response, 1L);
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("test");
+        IndexService test = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
         ShardPath shardPath = shard.shardPath();
         Path dataPath = shardPath.getDataPath();
@@ -431,8 +542,8 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         Path idxPath = env.sharedDataFile().resolve(randomAsciiOfLength(10));
         logger.info("--> idxPath: [{}]", idxPath);
         Settings idxSettings = Settings.builder()
-                .put(IndexMetaData.SETTING_DATA_PATH, idxPath)
-                .build();
+            .put(IndexMetaData.SETTING_DATA_PATH, idxPath)
+            .build();
         createIndex("test", idxSettings);
         ensureGreen("test");
         client().prepareIndex("test", "bar", "1").setSource("{}").setRefresh(true).get();
@@ -444,7 +555,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
 
     public void testExpectedShardSizeIsPresent() throws InterruptedException {
         assertAcked(client().admin().indices().prepareCreate("test")
-                .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0));
+            .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0));
         for (int i = 0; i < 50; i++) {
             client().prepareIndex("test", "test").setSource("{}").get();
         }
@@ -472,11 +583,11 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IOUtils.rm(endDir);
 
         Settings sb = Settings.builder()
-                .put(IndexMetaData.SETTING_DATA_PATH, startDir.toAbsolutePath().toString())
-                .build();
+            .put(IndexMetaData.SETTING_DATA_PATH, startDir.toAbsolutePath().toString())
+            .build();
         Settings sb2 = Settings.builder()
-                .put(IndexMetaData.SETTING_DATA_PATH, endDir.toAbsolutePath().toString())
-                .build();
+            .put(IndexMetaData.SETTING_DATA_PATH, endDir.toAbsolutePath().toString())
+            .build();
 
         logger.info("--> creating an index with data_path [{}]", startDir.toAbsolutePath().toString());
         createIndex(INDEX, sb);
@@ -507,9 +618,9 @@ public class IndexShardTests extends ESSingleNodeTestCase {
 
         logger.info("--> updating settings...");
         client().admin().indices().prepareUpdateSettings(INDEX)
-                .setSettings(sb2)
-                .setIndicesOptions(IndicesOptions.fromOptions(true, false, true, true))
-                .get();
+            .setSettings(sb2)
+            .setIndicesOptions(IndicesOptions.fromOptions(true, false, true, true))
+            .get();
 
         assert Files.exists(startDir) == false : "start dir shouldn't exist";
 
@@ -530,9 +641,9 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         createIndex("test");
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("test");
+        IndexService test = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
-        ShardStats stats = new ShardStats(shard.routingEntry(), shard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), shard, new CommonStatsFlags()), shard.commitStats());
+        ShardStats stats = new ShardStats(shard.routingEntry(), shard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), test.cache().getPercolatorQueryCache(), shard, new CommonStatsFlags()), shard.commitStats());
         assertEquals(shard.shardPath().getRootDataPath().toString(), stats.getDataPath());
         assertEquals(shard.shardPath().getRootStatePath().toString(), stats.getStatePath());
         assertEquals(shard.shardPath().isCustomDataPath(), stats.isCustomDataPath());
@@ -570,7 +681,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         ensureGreen();
         client().prepareIndex("test_iol", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefresh(true).get();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("test_iol");
+        IndexService test = indicesService.indexService(resolveIndex("test_iol"));
         IndexShard shard = test.getShardOrNull(0);
         AtomicInteger preIndex = new AtomicInteger();
         AtomicInteger postIndex = new AtomicInteger();
@@ -639,7 +750,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         try {
             shard.index(index);
             fail();
-        }catch (IllegalIndexShardStateException e){
+        } catch (IllegalIndexShardStateException e) {
 
         }
 
@@ -652,7 +763,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         try {
             shard.delete(delete);
             fail();
-        }catch (IllegalIndexShardStateException e){
+        } catch (IllegalIndexShardStateException e) {
 
         }
 
@@ -669,7 +780,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         createIndex("test", settingsBuilder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.REQUEST).build());
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("test");
+        IndexService test = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
         assertFalse(shard.shouldFlush());
         client().admin().indices().prepareUpdateSettings("test").setSettings(settingsBuilder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(133 /* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get();
@@ -689,7 +800,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         long size = shard.getEngine().getTranslog().sizeInBytes();
         logger.info("--> current translog size: [{}] num_ops [{}] generation [{}]", shard.getEngine().getTranslog().sizeInBytes(), shard.getEngine().getTranslog().totalOperations(), shard.getEngine().getTranslog().getGeneration());
         client().admin().indices().prepareUpdateSettings("test").setSettings(settingsBuilder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(size, ByteSizeUnit.BYTES))
-                .build()).get();
+            .build()).get();
         client().prepareDelete("test", "test", "2").get();
         logger.info("--> translog size after delete: [{}] num_ops [{}] generation [{}]", shard.getEngine().getTranslog().sizeInBytes(), shard.getEngine().getTranslog().totalOperations(), shard.getEngine().getTranslog().getGeneration());
         assertBusy(() -> { // this is async
@@ -703,7 +814,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         createIndex("test");
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("test");
+        IndexService test = indicesService.indexService(resolveIndex("test"));
         final IndexShard shard = test.getShardOrNull(0);
         assertFalse(shard.shouldFlush());
         client().admin().indices().prepareUpdateSettings("test").setSettings(settingsBuilder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(133/* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get();
@@ -749,7 +860,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         ).get());
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("test");
+        IndexService test = indicesService.indexService(resolveIndex("test"));
         final IndexShard shard = test.getShardOrNull(0);
         CountDownLatch latch = new CountDownLatch(1);
         Thread recoveryThread = new Thread(() -> {
@@ -779,7 +890,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         ).get());
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("test");
+        IndexService test = indicesService.indexService(resolveIndex("test"));
         final IndexShard shard = test.getShardOrNull(0);
         final int numThreads = randomIntBetween(2, 4);
         Thread[] indexThreads = new Thread[numThreads];
@@ -830,7 +941,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         createIndex("test");
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("test");
+        IndexService test = indicesService.indexService(resolveIndex("test"));
         final IndexShard shard = test.getShardOrNull(0);
         int translogOps = 1;
         client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get();
@@ -861,7 +972,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         createIndex("test");
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("test");
+        IndexService test = indicesService.indexService(resolveIndex("test"));
         final IndexShard shard = test.getShardOrNull(0);
         client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get();
         if (randomBoolean()) {
@@ -874,7 +985,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         newShard.updateRoutingEntry(routing, false);
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, Version.CURRENT);
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode,
-                localNode));
+            localNode));
         assertTrue(newShard.recoverFromStore(localNode));
         assertEquals(0, newShard.recoveryState().getTranslog().recoveredOperations());
         assertEquals(0, newShard.recoveryState().getTranslog().totalOperations());
@@ -887,12 +998,12 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertHitCount(response, 0);
     }
 
-    public void testFailIfIndexNotPresentInRecoverFromStore() throws IOException {
+    public void testFailIfIndexNotPresentInRecoverFromStore() throws Exception {
         createIndex("test");
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, Version.CURRENT);
-        IndexService test = indicesService.indexService("test");
+        IndexService test = indicesService.indexService(resolveIndex("test"));
         final IndexShard shard = test.getShardOrNull(0);
 
         client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get();
@@ -904,7 +1015,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         Store store = shard.store();
         store.incRef();
         test.removeShard(0, "b/c simon says so");
-        Lucene.cleanLuceneIndex(store.directory());
+        cleanLuceneIndex(store.directory());
         store.decRef();
         ShardRoutingHelper.reinit(routing);
         IndexShard newShard = test.createShard(routing);
@@ -937,7 +1048,12 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         newShard.updateRoutingEntry(routing, true);
         SearchResponse response = client().prepareSearch().get();
         assertHitCount(response, 0);
-        client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(true).get();
+        // we can't issue this request through a client because of the inconsistencies we created with the cluster state
+        // doing it directly instead
+        IndexRequest request = client().prepareIndex("test", "test", "0").setSource("{}").request();
+        request.process(MetaData.builder().put(test.getMetaData(), false).build(), null, false, "test");
+        TransportIndexAction.executeIndexRequestOnPrimary(request, newShard, null);
+        newShard.refresh("test");
         assertHitCount(client().prepareSearch().get(), 1);
     }
 
@@ -945,7 +1061,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         createIndex("test");
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("test");
+        IndexService test = indicesService.indexService(resolveIndex("test"));
         final IndexShard shard = test.getShardOrNull(0);
         ShardRouting origRouting = shard.routingEntry();
         assertThat(shard.state(), equalTo(IndexShardState.STARTED));
@@ -967,8 +1083,8 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         createIndex("test_target");
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("test");
-        IndexService test_target = indicesService.indexService("test_target");
+        IndexService test = indicesService.indexService(resolveIndex("test"));
+        IndexService test_target = indicesService.indexService(resolveIndex("test_target"));
         final IndexShard test_shard = test.getShardOrNull(0);
 
         client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get();
@@ -996,7 +1112,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             @Override
             public void restore(SnapshotId snapshotId, Version version, ShardId shardId, ShardId snapshotShardId, RecoveryState recoveryState) {
                 try {
-                    Lucene.cleanLuceneIndex(targetStore.directory());
+                    cleanLuceneIndex(targetStore.directory());
                     for (String file : sourceStore.directory().listAll()) {
                         if (file.equals("write.lock") || file.startsWith("extra")) {
                             continue;
@@ -1029,7 +1145,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         createIndex("test");
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService indexService = indicesService.indexService("test");
+        IndexService indexService = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = indexService.getShardOrNull(0);
         client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefresh(true).get();
         client().prepareIndex("test", "test", "1").setSource("{\"foobar\" : \"bar\"}").setRefresh(true).get();
@@ -1078,8 +1194,9 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         createIndex("test");
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService indexService = indicesService.indexService("test");
+        IndexService indexService = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = indexService.getShardOrNull(0);
+        client().admin().indices().preparePutMapping("test").setType("test").setSource("foo", "type=text,fielddata=true").get();
         client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefresh(true).get();
         client().prepareIndex("test", "test", "1").setSource("{\"foobar\" : \"bar\"}").setRefresh(true).get();
 
@@ -1126,7 +1243,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         createIndex("test");
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService indexService = indicesService.indexService("test");
+        IndexService indexService = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = indexService.getShardOrNull(0);
         client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefresh(true).get();
         IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {
@@ -1155,8 +1272,11 @@ public class IndexShardTests extends ESSingleNodeTestCase {
     private final IndexShard reinitWithWrapper(IndexService indexService, IndexShard shard, IndexSearcherWrapper wrapper, IndexingOperationListener... listeners) throws IOException {
         ShardRouting routing = new ShardRouting(shard.routingEntry());
         shard.close("simon says", true);
-        NodeServicesProvider indexServices = indexService.getIndexServices();
-        IndexShard newShard = new IndexShard(shard.shardId(), indexService.getIndexSettings(), shard.shardPath(), shard.store(), indexService.cache(), indexService.mapperService(), indexService.similarityService(), indexService.fieldData(), shard.getEngineFactory(), indexService.getIndexEventListener(), wrapper, indexServices, indexService.getSearchSlowLog(), null, listeners);
+        IndexShard newShard = new IndexShard(shard.shardId(), indexService.getIndexSettings(), shard.shardPath(),
+                shard.store(), indexService.cache(), indexService.mapperService(), indexService.similarityService(),
+                indexService.fieldData(), shard.getEngineFactory(), indexService.getIndexEventListener(), wrapper,
+                indexService.getThreadPool(), indexService.getBigArrays(), indexService.getSearchSlowLog(), null, listeners
+        );
         ShardRoutingHelper.reinit(routing);
         newShard.updateRoutingEntry(routing, false);
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, Version.CURRENT);
@@ -1179,7 +1299,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             .endObject().endObject().endObject()).get();
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("testindexfortranslogsync");
+        IndexService test = indicesService.indexService(resolveIndex("testindexfortranslogsync"));
         IndexShard shard = test.getShardOrNull(0);
         ShardRouting routing = new ShardRouting(shard.routingEntry());
         test.removeShard(0, "b/c britta says so");
@@ -1198,15 +1318,15 @@ public class IndexShardTests extends ESSingleNodeTestCase {
     public void testIndexingBufferDuringInternalRecovery() throws IOException {
         createIndex("index");
         client().admin().indices().preparePutMapping("index").setType("testtype").setSource(jsonBuilder().startObject()
-                .startObject("testtype")
-                .startObject("properties")
-                .startObject("foo")
-                .field("type", "text")
-                .endObject()
-                .endObject().endObject().endObject()).get();
+            .startObject("testtype")
+            .startObject("properties")
+            .startObject("foo")
+            .field("type", "text")
+            .endObject()
+            .endObject().endObject().endObject()).get();
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("index");
+        IndexService test = indicesService.indexService(resolveIndex("index"));
         IndexShard shard = test.getShardOrNull(0);
         ShardRouting routing = new ShardRouting(shard.routingEntry());
         test.removeShard(0, "b/c britta says so");
@@ -1227,15 +1347,15 @@ public class IndexShardTests extends ESSingleNodeTestCase {
     public void testIndexingBufferDuringPeerRecovery() throws IOException {
         createIndex("index");
         client().admin().indices().preparePutMapping("index").setType("testtype").setSource(jsonBuilder().startObject()
-                .startObject("testtype")
-                .startObject("properties")
-                .startObject("foo")
-                .field("type", "text")
-                .endObject()
-                .endObject().endObject().endObject()).get();
+            .startObject("testtype")
+            .startObject("properties")
+            .startObject("foo")
+            .field("type", "text")
+            .endObject()
+            .endObject().endObject().endObject()).get();
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("index");
+        IndexService test = indicesService.indexService(resolveIndex("index"));
         IndexShard shard = test.getShardOrNull(0);
         ShardRouting routing = new ShardRouting(shard.routingEntry());
         test.removeShard(0, "b/c britta says so");
diff --git a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java
index 2a52e8c557c..537ce83d3d2 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java
@@ -24,6 +24,7 @@ import org.elasticsearch.cluster.routing.AllocationId;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 
@@ -42,13 +43,13 @@ public class ShardPathTests extends ESTestCase {
             Settings.Builder builder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF")
                     .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
             Settings settings = builder.build();
-            ShardId shardId = new ShardId("foo", "_na_", 0);
+            ShardId shardId = new ShardId("foo", "0xDEADBEEF", 0);
             Path[] paths = env.availableShardPaths(shardId);
             Path path = randomFrom(paths);
             ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF", AllocationId.newInitializing()), 2, path);
             ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings));
             assertEquals(path, shardPath.getDataPath());
-            assertEquals("0xDEADBEEF", shardPath.getIndexUUID());
+            assertEquals("0xDEADBEEF", shardPath.getShardId().getIndex().getUUID());
             assertEquals("foo", shardPath.getShardId().getIndexName());
             assertEquals(path.resolve("translog"), shardPath.resolveTranslog());
             assertEquals(path.resolve("index"), shardPath.resolveIndex());
@@ -57,14 +58,15 @@ public class ShardPathTests extends ESTestCase {
 
     public void testFailLoadShardPathOnMultiState() throws IOException {
         try (final NodeEnvironment env = newNodeEnvironment(settingsBuilder().build())) {
-            Settings.Builder builder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF")
+            final String indexUUID = "0xDEADBEEF";
+            Settings.Builder builder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, indexUUID)
                     .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
             Settings settings = builder.build();
-            ShardId shardId = new ShardId("foo", "_na_", 0);
+            ShardId shardId = new ShardId("foo", indexUUID, 0);
             Path[] paths = env.availableShardPaths(shardId);
             assumeTrue("This test tests multi data.path but we only got one", paths.length > 1);
             int id = randomIntBetween(1, 10);
-            ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, "0xDEADBEEF", AllocationId.newInitializing()), id, paths);
+            ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, indexUUID, AllocationId.newInitializing()), id, paths);
             ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings));
             fail("Expected IllegalStateException");
         } catch (IllegalStateException e) {
@@ -77,7 +79,7 @@ public class ShardPathTests extends ESTestCase {
             Settings.Builder builder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, "foobar")
                     .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
             Settings settings = builder.build();
-            ShardId shardId = new ShardId("foo", "_na_", 0);
+            ShardId shardId = new ShardId("foo", "foobar", 0);
             Path[] paths = env.availableShardPaths(shardId);
             Path path = randomFrom(paths);
             int id = randomIntBetween(1, 10);
@@ -90,9 +92,10 @@ public class ShardPathTests extends ESTestCase {
     }
 
     public void testIllegalCustomDataPath() {
-        final Path path = createTempDir().resolve("foo").resolve("0");
+        Index index = new Index("foo", "foo");
+        final Path path = createTempDir().resolve(index.getUUID()).resolve("0");
         try {
-            new ShardPath(true, path, path, "foo", new ShardId("foo", "_na_", 0));
+            new ShardPath(true, path, path, new ShardId(index, 0));
             fail("Expected IllegalArgumentException");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), is("shard state path must be different to the data path when using custom data paths"));
@@ -100,8 +103,9 @@ public class ShardPathTests extends ESTestCase {
     }
 
     public void testValidCtor() {
-        final Path path = createTempDir().resolve("foo").resolve("0");
-        ShardPath shardPath = new ShardPath(false, path, path, "foo", new ShardId("foo", "_na_", 0));
+        Index index = new Index("foo", "foo");
+        final Path path = createTempDir().resolve(index.getUUID()).resolve("0");
+        ShardPath shardPath = new ShardPath(false, path, path, new ShardId(index, 0));
         assertFalse(shardPath.isCustomDataPath());
         assertEquals(shardPath.getDataPath(), path);
         assertEquals(shardPath.getShardStatePath(), path);
@@ -111,8 +115,9 @@ public class ShardPathTests extends ESTestCase {
         boolean useCustomDataPath = randomBoolean();
         final Settings indexSettings;
         final Settings nodeSettings;
+        final String indexUUID = "0xDEADBEEF";
         Settings.Builder indexSettingsBuilder = settingsBuilder()
-                .put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF")
+                .put(IndexMetaData.SETTING_INDEX_UUID, indexUUID)
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
         final Path customPath;
         if (useCustomDataPath) {
@@ -132,10 +137,10 @@ public class ShardPathTests extends ESTestCase {
             nodeSettings = Settings.EMPTY;
         }
         try (final NodeEnvironment env = newNodeEnvironment(nodeSettings)) {
-            ShardId shardId = new ShardId("foo", "_na_", 0);
+            ShardId shardId = new ShardId("foo", indexUUID, 0);
             Path[] paths = env.availableShardPaths(shardId);
             Path path = randomFrom(paths);
-            ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF", AllocationId.newInitializing()), 2, path);
+            ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, indexUUID, AllocationId.newInitializing()), 2, path);
             ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), indexSettings));
             boolean found = false;
             for (Path p : env.nodeDataPaths()) {
diff --git a/core/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java b/core/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java
new file mode 100644
index 00000000000..edb337fd4e6
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.similarity;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+
+import java.util.Collections;
+
+public class SimilarityServiceTests extends ESTestCase {
+
+    // Tests #16594
+    public void testOverrideBuiltInSimilarity() {
+        Settings settings = Settings.builder().put("index.similarity.BM25.type", "classic").build();
+        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", settings);
+        try {
+            new SimilarityService(indexSettings, Collections.emptyMap());
+            fail("can't override bm25");
+        } catch (IllegalArgumentException ex) {
+            assertEquals(ex.getMessage(), "Cannot redefine built-in Similarity [BM25]");
+        }
+    }
+
+    // Pre v3 indices could override built-in similarities
+    public void testOverrideBuiltInSimilarityPreV3() {
+        Settings settings = Settings.builder()
+                                    .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_0_0)
+                                    .put("index.similarity.BM25.type", "classic")
+                                    .build();
+        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", settings);
+        SimilarityService service = new SimilarityService(indexSettings, Collections.emptyMap());
+        assertTrue(service.getSimilarity("BM25") instanceof ClassicSimilarityProvider);
+    }
+
+    // Tests #16594
+    public void testDefaultSimilarity() {
+        Settings settings = Settings.builder().put("index.similarity.default.type", "BM25").build();
+        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", settings);
+        SimilarityService service = new SimilarityService(indexSettings, Collections.emptyMap());
+        assertTrue(service.getDefaultSimilarity() instanceof BM25SimilarityProvider);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
index 4031aa5da25..aaa1671f84b 100644
--- a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
+++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
@@ -49,6 +49,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.gateway.PrimaryShardAllocator;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.MergePolicyConfig;
 import org.elasticsearch.index.shard.IndexEventListener;
@@ -571,8 +572,9 @@ public class CorruptedFileIT extends ESIntegTestCase {
     private Map<String, List<Path>> findFilesToCorruptForReplica() throws IOException {
         Map<String, List<Path>> filesToNodes = new HashMap<>();
         ClusterState state = client().admin().cluster().prepareState().get().getState();
+        Index test = state.metaData().index("test").getIndex();
         for (ShardRouting shardRouting : state.getRoutingTable().allShards("test")) {
-            if (shardRouting.primary() == true) {
+            if (shardRouting.primary()) {
                 continue;
             }
             assertTrue(shardRouting.assignedToNode());
@@ -582,8 +584,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
             filesToNodes.put(nodeStats.getNode().getName(), files);
             for (FsInfo.Path info : nodeStats.getFs()) {
                 String path = info.getPath();
-                final String relativeDataLocationPath = "indices/test/" + Integer.toString(shardRouting.getId()) + "/index";
-                Path file = PathUtils.get(path).resolve(relativeDataLocationPath);
+                Path file = PathUtils.get(path).resolve("indices").resolve(test.getUUID()).resolve(Integer.toString(shardRouting.getId())).resolve("index");
                 if (Files.exists(file)) { // multi data path might only have one path in use
                     try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
                         for (Path item : stream) {
@@ -604,6 +605,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
 
     private ShardRouting corruptRandomPrimaryFile(final boolean includePerCommitFiles) throws IOException {
         ClusterState state = client().admin().cluster().prepareState().get().getState();
+        Index test = state.metaData().index("test").getIndex();
         GroupShardsIterator shardIterators = state.getRoutingNodes().getRoutingTable().activePrimaryShardsGrouped(new String[]{"test"}, false);
         List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
         ShardIterator shardIterator = RandomPicks.randomFrom(getRandom(), iterators);
@@ -616,8 +618,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
         Set<Path> files = new TreeSet<>(); // treeset makes sure iteration order is deterministic
         for (FsInfo.Path info : nodeStatses.getNodes()[0].getFs()) {
             String path = info.getPath();
-            final String relativeDataLocationPath = "indices/test/" + Integer.toString(shardRouting.getId()) + "/index";
-            Path file = PathUtils.get(path).resolve(relativeDataLocationPath);
+            Path file = PathUtils.get(path).resolve("indices").resolve(test.getUUID()).resolve(Integer.toString(shardRouting.getId())).resolve("index");
             if (Files.exists(file)) { // multi data path might only have one path in use
                 try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
                     for (Path item : stream) {
@@ -676,12 +677,13 @@ public class CorruptedFileIT extends ESIntegTestCase {
 
     public List<Path> listShardFiles(ShardRouting routing) throws IOException {
         NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(routing.currentNodeId()).setFs(true).get();
-
+        ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final Index test = state.metaData().index("test").getIndex();
         assertThat(routing.toString(), nodeStatses.getNodes().length, equalTo(1));
         List<Path> files = new ArrayList<>();
         for (FsInfo.Path info : nodeStatses.getNodes()[0].getFs()) {
             String path = info.getPath();
-            Path file = PathUtils.get(path).resolve("indices/test/" + Integer.toString(routing.getId()) + "/index");
+            Path file = PathUtils.get(path).resolve("indices/" + test.getUUID() + "/" + Integer.toString(routing.getId()) + "/index");
             if (Files.exists(file)) { // multi data path might only have one path in use
                 try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
                     for (Path item : stream) {
diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
index ae158b87c59..0a8cd9a6fe0 100644
--- a/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
+++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
@@ -33,6 +33,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.MockEngineFactoryPlugin;
 import org.elasticsearch.monitor.fs.FsInfo;
@@ -110,6 +111,7 @@ public class CorruptedTranslogIT extends ESIntegTestCase {
     private void corruptRandomTranslogFiles() throws IOException {
         ClusterState state = client().admin().cluster().prepareState().get().getState();
         GroupShardsIterator shardIterators = state.getRoutingNodes().getRoutingTable().activePrimaryShardsGrouped(new String[]{"test"}, false);
+        final Index test = state.metaData().index("test").getIndex();
         List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
         ShardIterator shardIterator = RandomPicks.randomFrom(getRandom(), iterators);
         ShardRouting shardRouting = shardIterator.nextOrNull();
@@ -121,7 +123,7 @@ public class CorruptedTranslogIT extends ESIntegTestCase {
         Set<Path> files = new TreeSet<>(); // treeset makes sure iteration order is deterministic
         for (FsInfo.Path fsPath : nodeStatses.getNodes()[0].getFs()) {
             String path = fsPath.getPath();
-            final String relativeDataLocationPath =  "indices/test/" + Integer.toString(shardRouting.getId()) + "/translog";
+            final String relativeDataLocationPath =  "indices/"+ test.getUUID() +"/" + Integer.toString(shardRouting.getId()) + "/translog";
             Path file = PathUtils.get(path).resolve(relativeDataLocationPath);
             if (Files.exists(file)) {
                 logger.info("--> path: {}", file);
diff --git a/core/src/test/java/org/elasticsearch/index/store/FsDirectoryServiceTests.java b/core/src/test/java/org/elasticsearch/index/store/FsDirectoryServiceTests.java
index 9da39b8da71..f7d793f03ed 100644
--- a/core/src/test/java/org/elasticsearch/index/store/FsDirectoryServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/store/FsDirectoryServiceTests.java
@@ -46,9 +46,9 @@ public class FsDirectoryServiceTests extends ESTestCase {
         IndexSettings settings = IndexSettingsModule.newIndexSettings("foo", build);
         IndexStoreConfig config = new IndexStoreConfig(build);
         IndexStore store = new IndexStore(settings, config);
-        Path tempDir = createTempDir().resolve("foo").resolve("0");
+        Path tempDir = createTempDir().resolve(settings.getUUID()).resolve("0");
         Files.createDirectories(tempDir);
-        ShardPath path = new ShardPath(false, tempDir, tempDir, settings.getUUID(), new ShardId(settings.getIndex(), 0));
+        ShardPath path = new ShardPath(false, tempDir, tempDir, new ShardId(settings.getIndex(), 0));
         FsDirectoryService fsDirectoryService = new FsDirectoryService(settings, store, path);
         Directory directory = fsDirectoryService.newDirectory();
         assertTrue(directory instanceof RateLimitedFSDirectory);
@@ -62,9 +62,9 @@ public class FsDirectoryServiceTests extends ESTestCase {
         IndexSettings settings = IndexSettingsModule.newIndexSettings("foo", build);
         IndexStoreConfig config = new IndexStoreConfig(build);
         IndexStore store = new IndexStore(settings, config);
-        Path tempDir = createTempDir().resolve("foo").resolve("0");
+        Path tempDir = createTempDir().resolve(settings.getUUID()).resolve("0");
         Files.createDirectories(tempDir);
-        ShardPath path = new ShardPath(false, tempDir, tempDir, settings.getUUID(), new ShardId(settings.getIndex(), 0));
+        ShardPath path = new ShardPath(false, tempDir, tempDir, new ShardId(settings.getIndex(), 0));
         FsDirectoryService fsDirectoryService = new FsDirectoryService(settings, store, path);
         Directory directory = fsDirectoryService.newDirectory();
         assertTrue(directory instanceof RateLimitedFSDirectory);
diff --git a/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java b/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java
index d9000e23a61..da5c1f3ecfa 100644
--- a/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java
@@ -47,13 +47,14 @@ import java.util.Locale;
 public class IndexStoreTests extends ESTestCase {
 
     public void testStoreDirectory() throws IOException {
-        final Path tempDir = createTempDir().resolve("foo").resolve("0");
+        Index index = new Index("foo", "fooUUID");
+        final Path tempDir = createTempDir().resolve(index.getUUID()).resolve("0");
         final IndexModule.Type[] values = IndexModule.Type.values();
         final IndexModule.Type type = RandomPicks.randomFrom(random(), values);
         Settings settings = Settings.settingsBuilder().put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), type.name().toLowerCase(Locale.ROOT))
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
-        FsDirectoryService service = new FsDirectoryService(indexSettings, null, new ShardPath(false, tempDir, tempDir, "foo", new ShardId("foo", "_na_", 0)));
+        FsDirectoryService service = new FsDirectoryService(indexSettings, null, new ShardPath(false, tempDir, tempDir, new ShardId(index, 0)));
         try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
             switch (type) {
                 case NIOFS:
@@ -84,8 +85,9 @@ public class IndexStoreTests extends ESTestCase {
     }
 
     public void testStoreDirectoryDefault() throws IOException {
-        final Path tempDir = createTempDir().resolve("foo").resolve("0");
-        FsDirectoryService service = new FsDirectoryService(IndexSettingsModule.newIndexSettings("foo", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()), null, new ShardPath(false, tempDir, tempDir, "foo", new ShardId("foo", "_na_", 0)));
+        Index index = new Index("bar", "foo");
+        final Path tempDir = createTempDir().resolve(index.getUUID()).resolve("0");
+        FsDirectoryService service = new FsDirectoryService(IndexSettingsModule.newIndexSettings("bar", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()), null, new ShardPath(false, tempDir, tempDir, new ShardId(index, 0)));
         try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
             if (Constants.WINDOWS) {
                 assertTrue(directory.toString(), directory instanceof MMapDirectory || directory instanceof SimpleFSDirectory);
diff --git a/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java b/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java
index 23c9e223972..62dec09e906 100644
--- a/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java
+++ b/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java
@@ -22,13 +22,15 @@ package org.elasticsearch.index.suggest.stats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
-import org.elasticsearch.action.suggest.SuggestRequestBuilder;
-import org.elasticsearch.action.suggest.SuggestResponse;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.search.stats.SearchStats;
+import org.elasticsearch.search.suggest.SuggestBuilder;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -86,49 +88,50 @@ public class SuggestStatsIT extends ESIntegTestCase {
 
         long startTime = System.currentTimeMillis();
         for (int i = 0; i < suggestAllIdx; i++) {
-            SuggestResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSuggest(), i).get();
+            SearchResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSearch(), i).get();
             assertAllSuccessful(suggestResponse);
         }
         for (int i = 0; i < suggestIdx1; i++) {
-            SuggestResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSuggest("test1"), i).get();
+            SearchResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSearch("test1"), i).get();
             assertAllSuccessful(suggestResponse);
         }
         for (int i = 0; i < suggestIdx2; i++) {
-            SuggestResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSuggest("test2"), i).get();
+            SearchResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSearch("test2"), i).get();
             assertAllSuccessful(suggestResponse);
         }
         long endTime = System.currentTimeMillis();
 
         IndicesStatsResponse indicesStats = client().admin().indices().prepareStats().execute().actionGet();
+        final SearchStats.Stats suggest = indicesStats.getTotal().getSearch().getTotal();
 
         // check current
-        assertThat(indicesStats.getTotal().getSuggest().getCurrent(), equalTo(0L));
+        assertThat(suggest.getSuggestCurrent(), equalTo(0L));
 
         // check suggest count
-        assertThat(indicesStats.getTotal().getSuggest().getCount(), equalTo((long) (suggestAllIdx * totalShards + suggestIdx1 * shardsIdx1 + suggestIdx2 * shardsIdx2)));
-        assertThat(indicesStats.getIndices().get("test1").getTotal().getSuggest().getCount(), equalTo((long) ((suggestAllIdx + suggestIdx1) * shardsIdx1)));
-        assertThat(indicesStats.getIndices().get("test2").getTotal().getSuggest().getCount(), equalTo((long) ((suggestAllIdx + suggestIdx2) * shardsIdx2)));
+        assertThat(suggest.getSuggestCount(), equalTo((long) (suggestAllIdx * totalShards + suggestIdx1 * shardsIdx1 + suggestIdx2 * shardsIdx2)));
+        assertThat(indicesStats.getIndices().get("test1").getTotal().getSearch().getTotal().getSuggestCount(), equalTo((long) ((suggestAllIdx + suggestIdx1) * shardsIdx1)));
+        assertThat(indicesStats.getIndices().get("test2").getTotal().getSearch().getTotal().getSuggestCount(), equalTo((long) ((suggestAllIdx + suggestIdx2) * shardsIdx2)));
 
         logger.info("iter {}, iter1 {}, iter2 {}, {}", suggestAllIdx, suggestIdx1, suggestIdx2, endTime - startTime);
         // check suggest time
-        assertThat(indicesStats.getTotal().getSuggest().getTimeInMillis(), greaterThan(0L));
+        assertThat(suggest.getSuggestTimeInMillis(), greaterThan(0L));
         // the upperbound is num shards * total time since we do searches in parallel
-        assertThat(indicesStats.getTotal().getSuggest().getTimeInMillis(), lessThanOrEqualTo(totalShards * (endTime - startTime)));
+        assertThat(suggest.getSuggestTimeInMillis(), lessThanOrEqualTo(totalShards * (endTime - startTime)));
 
         NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().execute().actionGet();
         NodeStats[] nodes = nodeStats.getNodes();
         Set<String> nodeIdsWithIndex = nodeIdsWithIndex("test1", "test2");
         int num = 0;
         for (NodeStats stat : nodes) {
-            SuggestStats suggestStats = stat.getIndices().getSuggest();
+            SearchStats.Stats suggestStats = stat.getIndices().getSearch().getTotal();
             logger.info("evaluating {}", stat.getNode());
             if (nodeIdsWithIndex.contains(stat.getNode().getId())) {
-                assertThat(suggestStats.getCount(), greaterThan(0L));
-                assertThat(suggestStats.getTimeInMillis(), greaterThan(0L));
+                assertThat(suggestStats.getSuggestCount(), greaterThan(0L));
+                assertThat(suggestStats.getSuggestTimeInMillis(), greaterThan(0L));
                 num++;
             } else {
-                assertThat(suggestStats.getCount(), equalTo(0L));
-                assertThat(suggestStats.getTimeInMillis(), equalTo(0L));
+                assertThat(suggestStats.getSuggestCount(), equalTo(0L));
+                assertThat(suggestStats.getSuggestTimeInMillis(), equalTo(0L));
             }
         }
 
@@ -136,15 +139,16 @@ public class SuggestStatsIT extends ESIntegTestCase {
 
     }
 
-    private SuggestRequestBuilder addSuggestions(SuggestRequestBuilder request, int i) {
+    private SearchRequestBuilder addSuggestions(SearchRequestBuilder request, int i) {
+        final SuggestBuilder suggestBuilder = new SuggestBuilder();
         for (int s = 0; s < randomIntBetween(2, 10); s++) {
             if (randomBoolean()) {
-                request.addSuggestion(new PhraseSuggestionBuilder("s" + s).field("f").text("test" + i + " test" + (i - 1)));
+                suggestBuilder.addSuggestion("s" + s, new PhraseSuggestionBuilder("f").text("test" + i + " test" + (i - 1)));
             } else {
-                request.addSuggestion(new TermSuggestionBuilder("s" + s).field("f").text("test" + i));
+                suggestBuilder.addSuggestion("s" + s, new TermSuggestionBuilder("f").text("test" + i));
             }
         }
-        return request;
+        return request.suggest(suggestBuilder);
     }
 
     private Set<String> nodeIdsWithIndex(String... indices) {
diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index 337d91356b9..984908ad9d6 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -1491,7 +1491,7 @@ public class TranslogTests extends ESTestCase {
                 if (writtenOperations.size() != snapshot.totalOperations()) {
                     for (int i = 0; i < threadCount; i++) {
                         if (threadExceptions[i] != null) {
-                            threadExceptions[i].printStackTrace();
+                            logger.info("Translog exception", threadExceptions[i]);
                         }
                     }
                 }
diff --git a/core/src/test/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java b/core/src/test/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java
index 4853d59588b..905e45cf9f7 100644
--- a/core/src/test/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java
+++ b/core/src/test/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java
@@ -61,23 +61,23 @@ import static org.hamcrest.Matchers.nullValue;
 public class IndexLifecycleActionIT extends ESIntegTestCase {
     public void testIndexLifecycleActionsWith11Shards1Backup() throws Exception {
         Settings settings = settingsBuilder()
+                .put(indexSettings())
                 .put(SETTING_NUMBER_OF_SHARDS, 11)
                 .put(SETTING_NUMBER_OF_REPLICAS, 1)
-                .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "0s")
                 .build();
 
         // start one server
         logger.info("Starting sever1");
-        final String server_1 = internalCluster().startNode(settings);
+        final String server_1 = internalCluster().startNode();
         final String node1 = getLocalNodeId(server_1);
 
         logger.info("Creating index [test]");
-        CreateIndexResponse createIndexResponse = client().admin().indices().create(createIndexRequest("test")).actionGet();
+        CreateIndexResponse createIndexResponse = client().admin().indices().create(createIndexRequest("test").settings(settings)).actionGet();
         assertThat(createIndexResponse.isAcknowledged(), equalTo(true));
 
         logger.info("Running Cluster Health");
         ClusterHealthResponse clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet();
-        logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
+        logger.info("Done Cluster Health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));
 
@@ -87,12 +87,12 @@ public class IndexLifecycleActionIT extends ESIntegTestCase {
 
         logger.info("Starting server2");
         // start another server
-        String server_2 = internalCluster().startNode(settings);
+        String server_2 = internalCluster().startNode();
 
         // first wait for 2 nodes in the cluster
         logger.info("Running Cluster Health");
         clusterHealth = client().admin().cluster().health(clusterHealthRequest().waitForGreenStatus().waitForNodes("2")).actionGet();
-        logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
+        logger.info("Done Cluster Health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
         final String node2 = getLocalNodeId(server_2);
@@ -122,7 +122,7 @@ public class IndexLifecycleActionIT extends ESIntegTestCase {
 
         logger.info("Starting server3");
         // start another server
-        String server_3 = internalCluster().startNode(settings);
+        String server_3 = internalCluster().startNode();
 
         // first wait for 3 nodes in the cluster
         clusterHealth = client().admin().cluster().health(clusterHealthRequest().waitForGreenStatus().waitForNodes("3")).actionGet();
@@ -171,7 +171,7 @@ public class IndexLifecycleActionIT extends ESIntegTestCase {
         // verify health
         logger.info("Running Cluster Health");
         clusterHealth = client().admin().cluster().health(clusterHealthRequest().waitForGreenStatus().waitForNodes("2")).actionGet();
-        logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
+        logger.info("Done Cluster Health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
 
diff --git a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
index afb9673508a..3316b52be2c 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
@@ -161,7 +161,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
     public void testShardAdditionAndRemoval() {
         createIndex("test", Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 3).put(SETTING_NUMBER_OF_REPLICAS, 0).build());
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("test");
+        IndexService test = indicesService.indexService(resolveIndex("test"));
 
         MockController controller = new MockController(Settings.builder()
                 .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "4mb").build());
@@ -194,7 +194,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
 
         createIndex("test", Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 2).put(SETTING_NUMBER_OF_REPLICAS, 0).build());
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("test");
+        IndexService test = indicesService.indexService(resolveIndex("test"));
 
         MockController controller = new MockController(Settings.builder()
                 .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "5mb")
@@ -248,7 +248,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
     public void testThrottling() throws Exception {
         createIndex("test", Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 3).put(SETTING_NUMBER_OF_REPLICAS, 0).build());
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService test = indicesService.indexService("test");
+        IndexService test = indicesService.indexService(resolveIndex("test"));
 
         MockController controller = new MockController(Settings.builder()
                 .put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "4mb").build());
@@ -281,7 +281,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
         controller.assertNotThrottled(shard0);
         controller.assertThrottled(shard1);
 
-        System.out.println("TEST: now index more");
+        logger.info("--> Indexing more data");
 
         // More indexing to shard0
         controller.simulateIndexing(shard0);
@@ -316,7 +316,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
         ensureGreen();
 
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexService indexService = indicesService.indexService("index");
+        IndexService indexService = indicesService.indexService(resolveIndex("index"));
         IndexShard shard = indexService.getShardOrNull(0);
         assertNotNull(shard);
 
@@ -342,7 +342,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
             @Override
             protected long getIndexBufferRAMBytesUsed(IndexShard shard) {
                 return shard.getIndexBufferRAMBytesUsed();
-            }   
+            }
 
             @Override
             protected void writeIndexingBufferAsync(IndexShard shard) {
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
index e34e1d6bd6b..367f4cd46ce 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
@@ -49,8 +49,9 @@ public class IndicesLifecycleListenerSingleNodeTests extends ESSingleNodeTestCas
         assertAcked(client().admin().indices().prepareCreate("test")
                 .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0));
         ensureGreen();
-        IndexMetaData metaData = indicesService.indexService("test").getMetaData();
-        ShardRouting shardRouting = indicesService.indexService("test").getShard(0).routingEntry();
+        Index idx = resolveIndex("test");
+        IndexMetaData metaData = indicesService.indexService(idx).getMetaData();
+        ShardRouting shardRouting = indicesService.indexService(idx).getShard(0).routingEntry();
         final AtomicInteger counter = new AtomicInteger(1);
         IndexEventListener countingListener = new IndexEventListener() {
             @Override
@@ -89,10 +90,11 @@ public class IndicesLifecycleListenerSingleNodeTests extends ESSingleNodeTestCas
                 counter.incrementAndGet();
             }
         };
-        indicesService.deleteIndex("test", "simon says");
+        indicesService.deleteIndex(idx, "simon says");
         try {
             NodeServicesProvider nodeServicesProvider = getInstanceFromNode(NodeServicesProvider.class);
             IndexService index = indicesService.createIndex(nodeServicesProvider, metaData, Arrays.asList(countingListener));
+            idx = index.index();
             ShardRouting newRouting = new ShardRouting(shardRouting);
             String nodeId = newRouting.currentNodeId();
             ShardRoutingHelper.moveToUnassigned(newRouting, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "boom"));
@@ -106,7 +108,7 @@ public class IndicesLifecycleListenerSingleNodeTests extends ESSingleNodeTestCas
             ShardRoutingHelper.moveToStarted(newRouting);
             shard.updateRoutingEntry(newRouting, true);
         } finally {
-            indicesService.deleteIndex("test", "simon says");
+            indicesService.deleteIndex(idx, "simon says");
         }
         assertEquals(7, counter.get());
     }
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
index 1d7d833c500..046d0b4400f 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
@@ -37,20 +37,22 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder;
 import org.elasticsearch.action.percolate.MultiPercolateRequestBuilder;
+import org.elasticsearch.action.percolate.MultiPercolateResponse;
 import org.elasticsearch.action.percolate.PercolateRequestBuilder;
 import org.elasticsearch.action.percolate.PercolateSourceBuilder;
 import org.elasticsearch.action.search.MultiSearchRequestBuilder;
 import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.suggest.SuggestRequestBuilder;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.suggest.SuggestBuilder;
 import org.elasticsearch.search.suggest.SuggestBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -62,8 +64,10 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.startsWith;
 
 public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
 
@@ -90,7 +94,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(typesExists("test1", "test2"), true);
         verify(percolate("test1", "test2"), true);
         verify(mpercolate(null, "test1", "test2"), false);
-        verify(suggest("test1", "test2"), true);
         verify(getAliases("test1", "test2"), true);
         verify(getFieldMapping("test1", "test2"), true);
         verify(getMapping("test1", "test2"), true);
@@ -110,7 +113,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(typesExists("test1", "test2").setIndicesOptions(options), true);
         verify(percolate("test1", "test2").setIndicesOptions(options), true);
         verify(mpercolate(options, "test1", "test2").setIndicesOptions(options), false);
-        verify(suggest("test1", "test2").setIndicesOptions(options), true);
         verify(getAliases("test1", "test2").setIndicesOptions(options), true);
         verify(getFieldMapping("test1", "test2").setIndicesOptions(options), true);
         verify(getMapping("test1", "test2").setIndicesOptions(options), true);
@@ -130,7 +132,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(typesExists("test1", "test2").setIndicesOptions(options), false);
         verify(percolate("test1", "test2").setIndicesOptions(options), false);
         verify(mpercolate(options, "test1", "test2").setIndicesOptions(options), false);
-        verify(suggest("test1", "test2").setIndicesOptions(options), false);
         verify(getAliases("test1", "test2").setIndicesOptions(options), false);
         verify(getFieldMapping("test1", "test2").setIndicesOptions(options), false);
         verify(getMapping("test1", "test2").setIndicesOptions(options), false);
@@ -152,7 +153,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(typesExists("test1", "test2").setIndicesOptions(options), false);
         verify(percolate("test1", "test2").setIndicesOptions(options), false);
         verify(mpercolate(options, "test1", "test2").setIndicesOptions(options), false);
-        verify(suggest("test1", "test2").setIndicesOptions(options), false);
         verify(getAliases("test1", "test2").setIndicesOptions(options), false);
         verify(getFieldMapping("test1", "test2").setIndicesOptions(options), false);
         verify(getMapping("test1", "test2").setIndicesOptions(options), false);
@@ -182,7 +182,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(typesExists("test1").setIndicesOptions(options), true);
         verify(percolate("test1").setIndicesOptions(options), true);
         verify(mpercolate(options, "test1").setIndicesOptions(options), true);
-        verify(suggest("test1").setIndicesOptions(options), true);
         verify(getAliases("test1").setIndicesOptions(options), true);
         verify(getFieldMapping("test1").setIndicesOptions(options), true);
         verify(getMapping("test1").setIndicesOptions(options), true);
@@ -202,7 +201,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(typesExists("test1").setIndicesOptions(options), false);
         verify(percolate("test1").setIndicesOptions(options), false);
         verify(mpercolate(options, "test1").setIndicesOptions(options), false);
-        verify(suggest("test1").setIndicesOptions(options), false);
         verify(getAliases("test1").setIndicesOptions(options), false);
         verify(getFieldMapping("test1").setIndicesOptions(options), false);
         verify(getMapping("test1").setIndicesOptions(options), false);
@@ -225,7 +223,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(typesExists("test1").setIndicesOptions(options), false);
         verify(percolate("test1").setIndicesOptions(options), false);
         verify(mpercolate(options, "test1").setIndicesOptions(options), false);
-        verify(suggest("test1").setIndicesOptions(options), false);
         verify(getAliases("test1").setIndicesOptions(options), false);
         verify(getFieldMapping("test1").setIndicesOptions(options), false);
         verify(getMapping("test1").setIndicesOptions(options), false);
@@ -246,7 +243,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(aliasExists("test1").setIndicesOptions(options), true);
         verify(typesExists("test1").setIndicesOptions(options), true);
         verify(percolate("test1").setIndicesOptions(options), true);
-        verify(suggest("test1").setIndicesOptions(options), true);
         verify(getAliases("test1").setIndicesOptions(options), true);
         verify(getFieldMapping("test1").setIndicesOptions(options), true);
         verify(getMapping("test1").setIndicesOptions(options), true);
@@ -265,7 +261,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(aliasExists("test1").setIndicesOptions(options), false);
         verify(typesExists("test1").setIndicesOptions(options), false);
         verify(percolate("test1").setIndicesOptions(options), false);
-        verify(suggest("test1").setIndicesOptions(options), false);
         verify(getAliases("test1").setIndicesOptions(options), false);
         verify(getFieldMapping("test1").setIndicesOptions(options), false);
         verify(getMapping("test1").setIndicesOptions(options), false);
@@ -287,7 +282,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(aliasExists("test1").setIndicesOptions(options), false);
         verify(typesExists("test1").setIndicesOptions(options), false);
         verify(percolate("test1").setIndicesOptions(options), false);
-        verify(suggest("test1").setIndicesOptions(options), false);
         verify(getAliases("test1").setIndicesOptions(options), false);
         verify(getFieldMapping("test1").setIndicesOptions(options), false);
         verify(getMapping("test1").setIndicesOptions(options), false);
@@ -340,7 +334,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(typesExists(indices), false);
         verify(percolate(indices), false);
         verify(mpercolate(null, indices), false);
-        verify(suggest(indices), false);
         verify(getAliases(indices), false);
         verify(getFieldMapping(indices), false);
         verify(getMapping(indices), false);
@@ -361,7 +354,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(typesExists(indices).setIndicesOptions(options), false);
         verify(percolate(indices).setIndicesOptions(options), false);
         verify(mpercolate(options, indices), false);
-        verify(suggest(indices).setIndicesOptions(options), false);
         verify(getAliases(indices).setIndicesOptions(options), false);
         verify(getFieldMapping(indices).setIndicesOptions(options), false);
         verify(getMapping(indices).setIndicesOptions(options), false);
@@ -385,7 +377,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(typesExists(indices), false);
         verify(percolate(indices), false);
         verify(mpercolate(null, indices), false);
-        verify(suggest(indices), false);
         verify(getAliases(indices), false);
         verify(getFieldMapping(indices), false);
         verify(getMapping(indices), false);
@@ -406,7 +397,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(typesExists(indices), false);
         verify(percolate(indices), false);
         verify(mpercolate(null, indices), false);
-        verify(suggest(indices), false);
         verify(getAliases(indices), false);
         verify(getFieldMapping(indices), false);
         verify(getMapping(indices), false);
@@ -427,7 +417,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(typesExists(indices).setIndicesOptions(options), false);
         verify(percolate(indices).setIndicesOptions(options), false);
         verify(mpercolate(options, indices), false);
-        verify(suggest(indices).setIndicesOptions(options), false);
         verify(getAliases(indices).setIndicesOptions(options), false);
         verify(getFieldMapping(indices).setIndicesOptions(options), false);
         verify(getMapping(indices).setIndicesOptions(options), false);
@@ -642,9 +631,12 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
             return "a plugin that adds a dynamic tst setting";
         }
 
-        private static final Setting<String> INDEX_A = new Setting<>("index.a", "", Function.identity(), true, Setting.Scope.INDEX);
-        private static final Setting<String> INDEX_C = new Setting<>("index.c", "", Function.identity(), true, Setting.Scope.INDEX);
-        private static final Setting<String> INDEX_E = new Setting<>("index.e", "", Function.identity(), false, Setting.Scope.INDEX);
+        private static final Setting<String> INDEX_A =
+            new Setting<>("index.a", "", Function.identity(), Property.Dynamic, Property.IndexScope);
+        private static final Setting<String> INDEX_C =
+            new Setting<>("index.c", "", Function.identity(), Property.Dynamic, Property.IndexScope);
+        private static final Setting<String> INDEX_E =
+            new Setting<>("index.e", "", Function.identity(), Property.IndexScope);
 
 
         public void onModule(SettingsModule module) {
@@ -681,7 +673,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         try {
             verify(client().admin().indices().prepareUpdateSettings("barbaz").setSettings(Settings.builder().put("e", "f")), false);
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("Can't update non dynamic settings[[index.e]] for open indices [[barbaz]]"));
+            assertThat(e.getMessage(), startsWith("Can't update non dynamic settings [[index.e]] for open indices [[barbaz"));
         }
         verify(client().admin().indices().prepareUpdateSettings("baz*").setSettings(Settings.builder().put("a", "b")), true);
     }
@@ -748,10 +740,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         return builder.add(percolate(indices));
     }
 
-    private static SuggestRequestBuilder suggest(String... indices) {
-        return client().prepareSuggest(indices).addSuggestion(SuggestBuilders.termSuggestion("name").field("a"));
-    }
-
     private static GetAliasesRequestBuilder getAliases(String... indices) {
         return client().admin().indices().prepareGetAliases("dummy").addIndices(indices);
     }
@@ -788,7 +776,13 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
             if (requestBuilder instanceof MultiSearchRequestBuilder) {
                 MultiSearchResponse multiSearchResponse = ((MultiSearchRequestBuilder) requestBuilder).get();
                 assertThat(multiSearchResponse.getResponses().length, equalTo(1));
+                assertThat(multiSearchResponse.getResponses()[0].isFailure(), is(true));
                 assertThat(multiSearchResponse.getResponses()[0].getResponse(), nullValue());
+            } else if (requestBuilder instanceof MultiPercolateRequestBuilder) {
+                MultiPercolateResponse multiPercolateResponse = ((MultiPercolateRequestBuilder) requestBuilder).get();
+                assertThat(multiPercolateResponse.getItems().length, equalTo(1));
+                assertThat(multiPercolateResponse.getItems()[0].isFailure(), is(true));
+                assertThat(multiPercolateResponse.getItems()[0].getResponse(), nullValue());
             } else {
                 try {
                     requestBuilder.get();
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
index 1b2ad9c0a1e..fec50cf0a27 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
@@ -21,18 +21,22 @@ package org.elasticsearch.indices;
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.indices.IndicesRequestCache;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
+import org.joda.time.chrono.ISOChronology;
 
 import java.util.List;
 
 import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
 public class IndicesRequestCacheIT extends ESIntegTestCase {
@@ -80,4 +84,274 @@ public class IndicesRequestCacheIT extends ESIntegTestCase {
         }
     }
 
+    public void testQueryRewrite() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("index").addMapping("type", "s", "type=text")
+                .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true,
+                        IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5,
+                        IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .get());
+        indexRandom(true, client().prepareIndex("index", "type", "1").setRouting("1").setSource("s", "a"),
+                client().prepareIndex("index", "type", "2").setRouting("1").setSource("s", "b"),
+                client().prepareIndex("index", "type", "3").setRouting("1").setSource("s", "c"),
+                client().prepareIndex("index", "type", "4").setRouting("2").setSource("s", "d"),
+                client().prepareIndex("index", "type", "5").setRouting("2").setSource("s", "e"),
+                client().prepareIndex("index", "type", "6").setRouting("2").setSource("s", "f"),
+                client().prepareIndex("index", "type", "7").setRouting("3").setSource("s", "g"),
+                client().prepareIndex("index", "type", "8").setRouting("3").setSource("s", "h"),
+                client().prepareIndex("index", "type", "9").setRouting("3").setSource("s", "i"));
+        ensureSearchable("index");
+
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(0L));
+
+        final SearchResponse r1 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("s").gte("a").lte("g")).get();
+        assertSearchResponse(r1);
+        assertThat(r1.getHits().getTotalHits(), equalTo(7L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(5L));
+
+        final SearchResponse r2 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("s").gte("b").lte("h")).get();
+        assertSearchResponse(r2);
+        assertThat(r2.getHits().getTotalHits(), equalTo(7L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(3L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(7L));
+
+        final SearchResponse r3 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("s").gte("c").lte("i")).get();
+        assertSearchResponse(r3);
+        assertThat(r3.getHits().getTotalHits(), equalTo(7L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(6L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(9L));
+    }
+
+    public void testQueryRewriteMissingValues() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("index").addMapping("type", "s", "type=text")
+                .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS,
+                        1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .get());
+        indexRandom(true, client().prepareIndex("index", "type", "1").setSource("s", "a"),
+                client().prepareIndex("index", "type", "2").setSource("s", "b"),
+                client().prepareIndex("index", "type", "3").setSource("s", "c"),
+                client().prepareIndex("index", "type", "4").setSource("s", "d"),
+                client().prepareIndex("index", "type", "5").setSource("s", "e"),
+                client().prepareIndex("index", "type", "6").setSource("s", "f"),
+                client().prepareIndex("index", "type", "7").setSource("other", "value"),
+                client().prepareIndex("index", "type", "8").setSource("s", "h"),
+                client().prepareIndex("index", "type", "9").setSource("s", "i"));
+        ensureSearchable("index");
+
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(0L));
+
+        final SearchResponse r1 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("s").gte("a").lte("j")).get();
+        assertSearchResponse(r1);
+        assertThat(r1.getHits().getTotalHits(), equalTo(8L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(1L));
+
+        final SearchResponse r2 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("s").gte("a").lte("j")).get();
+        assertSearchResponse(r2);
+        assertThat(r2.getHits().getTotalHits(), equalTo(8L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(1L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(1L));
+
+        final SearchResponse r3 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("s").gte("a").lte("j")).get();
+        assertSearchResponse(r3);
+        assertThat(r3.getHits().getTotalHits(), equalTo(8L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(2L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(1L));
+    }
+
+    public void testQueryRewriteDates() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("index").addMapping("type", "d", "type=date")
+                .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true,
+                        IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1,
+                        IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .get());
+        indexRandom(true, client().prepareIndex("index", "type", "1").setSource("d", "2014-01-01T00:00:00"),
+                client().prepareIndex("index", "type", "2").setSource("d", "2014-02-01T00:00:00"),
+                client().prepareIndex("index", "type", "3").setSource("d", "2014-03-01T00:00:00"),
+                client().prepareIndex("index", "type", "4").setSource("d", "2014-04-01T00:00:00"),
+                client().prepareIndex("index", "type", "5").setSource("d", "2014-05-01T00:00:00"),
+                client().prepareIndex("index", "type", "6").setSource("d", "2014-06-01T00:00:00"),
+                client().prepareIndex("index", "type", "7").setSource("d", "2014-07-01T00:00:00"),
+                client().prepareIndex("index", "type", "8").setSource("d", "2014-08-01T00:00:00"),
+                client().prepareIndex("index", "type", "9").setSource("d", "2014-09-01T00:00:00"));
+        ensureSearchable("index");
+
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(0L));
+
+        final SearchResponse r1 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
+                .get();
+        assertSearchResponse(r1);
+        assertThat(r1.getHits().getTotalHits(), equalTo(9L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(1L));
+
+        final SearchResponse r2 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
+                .get();
+        assertSearchResponse(r2);
+        assertThat(r2.getHits().getTotalHits(), equalTo(9L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(1L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(1L));
+
+        final SearchResponse r3 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
+                .get();
+        assertSearchResponse(r3);
+        assertThat(r3.getHits().getTotalHits(), equalTo(9L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(2L));
+        assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(1L));
+    }
+
+    public void testQueryRewriteDatesWithNow() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("index-1").addMapping("type", "d", "type=date")
+                .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS,
+                        1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .get());
+        assertAcked(client().admin().indices().prepareCreate("index-2").addMapping("type", "d", "type=date")
+                .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS,
+                        1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .get());
+        assertAcked(client().admin().indices().prepareCreate("index-3").addMapping("type", "d", "type=date")
+                .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS,
+                        1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .get());
+        DateTime now = new DateTime(ISOChronology.getInstanceUTC());
+        indexRandom(true, client().prepareIndex("index-1", "type", "1").setSource("d", now),
+                client().prepareIndex("index-1", "type", "2").setSource("d", now.minusDays(1)),
+                client().prepareIndex("index-1", "type", "3").setSource("d", now.minusDays(2)),
+                client().prepareIndex("index-2", "type", "4").setSource("d", now.minusDays(3)),
+                client().prepareIndex("index-2", "type", "5").setSource("d", now.minusDays(4)),
+                client().prepareIndex("index-2", "type", "6").setSource("d", now.minusDays(5)),
+                client().prepareIndex("index-3", "type", "7").setSource("d", now.minusDays(6)),
+                client().prepareIndex("index-3", "type", "8").setSource("d", now.minusDays(7)),
+                client().prepareIndex("index-3", "type", "9").setSource("d", now.minusDays(8)));
+        ensureSearchable("index-1", "index-2", "index-3");
+
+        assertThat(
+                client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(0L));
+        assertThat(
+                client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(0L));
+
+        assertThat(
+                client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(0L));
+        assertThat(
+                client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(0L));
+        assertThat(
+                client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(0L));
+        assertThat(
+                client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(0L));
+
+        final SearchResponse r1 = client().prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get();
+        assertSearchResponse(r1);
+        assertThat(r1.getHits().getTotalHits(), equalTo(8L));
+        assertThat(
+                client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(0L));
+        assertThat(
+                client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(1L));
+        assertThat(
+                client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(0L));
+        assertThat(
+                client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(1L));
+        // Because the query will INTERSECT with the 3rd index it will not be
+        // rewritten and will still contain `now` so won't be recorded as a
+        // cache miss or cache hit since queries containing now can't be cached
+        assertThat(
+                client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(0L));
+        assertThat(
+                client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(0L));
+
+        final SearchResponse r2 = client().prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get();
+        assertSearchResponse(r2);
+        assertThat(r2.getHits().getTotalHits(), equalTo(8L));
+        assertThat(
+                client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(1L));
+        assertThat(
+                client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(1L));
+        assertThat(
+                client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(1L));
+        assertThat(
+                client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(1L));
+        assertThat(
+                client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(0L));
+        assertThat(
+                client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(0L));
+
+        final SearchResponse r3 = client().prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
+                .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get();
+        assertSearchResponse(r3);
+        assertThat(r3.getHits().getTotalHits(), equalTo(8L));
+        assertThat(
+                client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(2L));
+        assertThat(
+                client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(1L));
+        assertThat(
+                client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(2L));
+        assertThat(
+                client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(1L));
+        assertThat(
+                client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
+                equalTo(0L));
+        assertThat(
+                client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
+                equalTo(0L));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
index e36f1bca49b..646d9651436 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
@@ -282,7 +282,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
         assertEquals("foo", value1.toUtf8());
         BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes());
         assertEquals("bar", value2.toUtf8());
-        logger.info(requestCacheStats.stats().getMemorySize().toString());
+        logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize());
         BytesReference value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes());
         assertEquals("baz", value3.toUtf8());
         assertEquals(2, cache.count());
@@ -319,7 +319,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
         assertEquals("foo", value1.toUtf8());
         BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes());
         assertEquals("bar", value2.toUtf8());
-        logger.info(requestCacheStats.stats().getMemorySize().toString());
+        logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize());
         BytesReference value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes());
         assertEquals("baz", value3.toUtf8());
         assertEquals(3, cache.count());
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
index e9f1f6be518..344c40a92ff 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
@@ -20,14 +20,13 @@ package org.elasticsearch.indices;
 
 import org.apache.lucene.store.LockObtainFailedException;
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.gateway.GatewayMetaState;
-import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.ShardId;
@@ -73,12 +72,16 @@ public class IndicesServiceTests extends ESSingleNodeTestCase {
         IndexMetaData meta = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(
                 1).build();
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", meta.getSettings());
-        assertFalse("no shard location", indicesService.canDeleteShardContent(new ShardId("test", "_na_", 0), indexSettings));
+        ShardId shardId = new ShardId(meta.getIndex(), 0);
+        assertFalse("no shard location", indicesService.canDeleteShardContent(shardId, indexSettings));
         IndexService test = createIndex("test");
+        shardId = new ShardId(test.index(), 0);
         assertTrue(test.hasShard(0));
-        assertFalse("shard is allocated", indicesService.canDeleteShardContent(new ShardId("test", "_na_", 0), indexSettings));
+        assertFalse("shard is allocated", indicesService.canDeleteShardContent(shardId, test.getIndexSettings()));
         test.removeShard(0, "boom");
-        assertTrue("shard is removed", indicesService.canDeleteShardContent(new ShardId("test", "_na_", 0), indexSettings));
+        assertTrue("shard is removed", indicesService.canDeleteShardContent(shardId, test.getIndexSettings()));
+        ShardId notAllocated = new ShardId(test.index(), 100);
+        assertFalse("shard that was never on this node should NOT be deletable", indicesService.canDeleteShardContent(notAllocated, test.getIndexSettings()));
     }
 
     public void testDeleteIndexStore() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java
index 239cb7a9096..da1bb7ae303 100644
--- a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java
@@ -18,10 +18,10 @@
  */
 package org.elasticsearch.indices.flush;
 
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.index.IndexService;
@@ -42,7 +42,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase {
     public void testModificationPreventsFlushing() throws InterruptedException {
         createIndex("test");
         client().prepareIndex("test", "test", "1").setSource("{}").get();
-        IndexService test = getInstanceFromNode(IndicesService.class).indexService("test");
+        IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
 
         SyncedFlushService flushService = getInstanceFromNode(SyncedFlushService.class);
@@ -86,7 +86,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase {
     public void testSingleShardSuccess() throws InterruptedException {
         createIndex("test");
         client().prepareIndex("test", "test", "1").setSource("{}").get();
-        IndexService test = getInstanceFromNode(IndicesService.class).indexService("test");
+        IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
 
         SyncedFlushService flushService = getInstanceFromNode(SyncedFlushService.class);
@@ -106,7 +106,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase {
     public void testSyncFailsIfOperationIsInFlight() throws InterruptedException {
         createIndex("test");
         client().prepareIndex("test", "test", "1").setSource("{}").get();
-        IndexService test = getInstanceFromNode(IndicesService.class).indexService("test");
+        IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
 
         SyncedFlushService flushService = getInstanceFromNode(SyncedFlushService.class);
@@ -126,7 +126,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase {
 
     public void testSyncFailsOnIndexClosedOrMissing() throws InterruptedException {
         createIndex("test");
-        IndexService test = getInstanceFromNode(IndicesService.class).indexService("test");
+        IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
 
         SyncedFlushService flushService = getInstanceFromNode(SyncedFlushService.class);
@@ -159,7 +159,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase {
     public void testFailAfterIntermediateCommit() throws InterruptedException {
         createIndex("test");
         client().prepareIndex("test", "test", "1").setSource("{}").get();
-        IndexService test = getInstanceFromNode(IndicesService.class).indexService("test");
+        IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
 
         SyncedFlushService flushService = getInstanceFromNode(SyncedFlushService.class);
@@ -192,7 +192,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase {
     public void testFailWhenCommitIsMissing() throws InterruptedException {
         createIndex("test");
         client().prepareIndex("test", "test", "1").setSource("{}").get();
-        IndexService test = getInstanceFromNode(IndicesService.class).indexService("test");
+        IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
 
         SyncedFlushService flushService = getInstanceFromNode(SyncedFlushService.class);
diff --git a/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java
index 0951f3c46df..8e064f46e12 100644
--- a/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java
@@ -156,15 +156,15 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase {
 
     public void testUpdateMappingWithNormsConflicts() throws Exception {
         client().admin().indices().prepareCreate("test")
-                .addMapping("type", "{\"type\":{\"properties\":{\"body\":{\"type\":\"text\", \"norms\": { \"enabled\": false }}}}}")
+                .addMapping("type", "{\"type\":{\"properties\":{\"body\":{\"type\":\"text\", \"norms\": false }}}}")
                 .execute().actionGet();
         try {
             client().admin().indices().preparePutMapping("test").setType("type")
-                    .setSource("{\"type\":{\"properties\":{\"body\":{\"type\":\"text\", \"norms\": { \"enabled\": true }}}}}").execute()
+                    .setSource("{\"type\":{\"properties\":{\"body\":{\"type\":\"text\", \"norms\": true }}}}").execute()
                     .actionGet();
             fail("Expected MergeMappingException");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("mapper [body] has different [omit_norms]"));
+            assertThat(e.getMessage(), containsString("mapper [body] has different [norms]"));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
index ec808647787..a7cc93f75c7 100644
--- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
@@ -148,7 +148,7 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
 
         // Create an index where the mappings have a field data filter
         assertAcked(prepareCreate("ramtest").setSource("{\"mappings\": {\"type\": {\"properties\": {\"test\": " +
-                "{\"type\": \"text\",\"fielddata\": {\"filter\": {\"regex\": {\"pattern\": \"^value.*\"}}}}}}}}"));
+                "{\"type\": \"text\",\"fielddata_frequency_filter\": {\"max\": 10000}}}}}}"));
 
         ensureGreen("ramtest");
 
diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
index 35ed7a2c657..87d6fcb116d 100644
--- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
@@ -31,6 +31,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.unit.TimeValue;
@@ -81,8 +82,6 @@ public class RandomExceptionCircuitBreakerIT extends ESIntegTestCase {
                 .startObject("test-num")
                         // I don't use randomNumericType() here because I don't want "byte", and I want "float" and "double"
                 .field("type", randomFrom(Arrays.asList("float", "long", "double", "short", "integer")))
-                .startObject("fielddata")
-                .endObject() // fielddata
                 .endObject() // test-num
                 .endObject() // properties
                 .endObject() // type
@@ -197,8 +196,10 @@ public class RandomExceptionCircuitBreakerIT extends ESIntegTestCase {
     // TODO: Generalize this class and add it as a utility
     public static class RandomExceptionDirectoryReaderWrapper extends MockEngineSupport.DirectoryReaderWrapper {
 
-        public static final Setting<Double> EXCEPTION_TOP_LEVEL_RATIO_SETTING = Setting.doubleSetting(EXCEPTION_TOP_LEVEL_RATIO_KEY, 0.1d, 0.0d, false, Setting.Scope.INDEX);
-        public static final Setting<Double> EXCEPTION_LOW_LEVEL_RATIO_SETTING = Setting.doubleSetting(EXCEPTION_LOW_LEVEL_RATIO_KEY, 0.1d, 0.0d, false, Setting.Scope.INDEX);
+        public static final Setting<Double> EXCEPTION_TOP_LEVEL_RATIO_SETTING =
+            Setting.doubleSetting(EXCEPTION_TOP_LEVEL_RATIO_KEY, 0.1d, 0.0d, Property.IndexScope);
+        public static final Setting<Double> EXCEPTION_LOW_LEVEL_RATIO_SETTING =
+            Setting.doubleSetting(EXCEPTION_LOW_LEVEL_RATIO_KEY, 0.1d, 0.0d, Property.IndexScope);
         public static class TestPlugin extends Plugin {
             @Override
             public String name() {
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java b/core/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java
index d105c98f4ac..322ed269bcf 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java
@@ -40,6 +40,7 @@ public class IndexPrimaryRelocationIT extends ESIntegTestCase {
 
     private static final int RELOCATION_COUNT = 25;
 
+    @TestLogging("_root:DEBUG,action.delete:TRACE,action.index:TRACE,index.shard:TRACE,cluster.service:TRACE")
     public void testPrimaryRelocationWhileIndexing() throws Exception {
         internalCluster().ensureAtLeastNumDataNodes(randomIntBetween(2, 3));
         client().admin().indices().prepareCreate("test")
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
index 155032f1d8c..140ff153b9f 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
@@ -30,13 +30,14 @@ import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.IndicesService;
@@ -261,14 +262,16 @@ public class IndexRecoveryIT extends ESIntegTestCase {
                 .execute().actionGet().getState();
 
         logger.info("--> waiting for recovery to start both on source and target");
+        final Index index = resolveIndex(INDEX_NAME);
         assertBusy(new Runnable() {
             @Override
             public void run() {
+
                 IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeA);
-                assertThat(indicesService.indexServiceSafe(INDEX_NAME).getShard(0).recoveryStats().currentAsSource(),
+                assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsSource(),
                         equalTo(1));
                 indicesService = internalCluster().getInstance(IndicesService.class, nodeB);
-                assertThat(indicesService.indexServiceSafe(INDEX_NAME).getShard(0).recoveryStats().currentAsTarget(),
+                assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsTarget(),
                         equalTo(1));
             }
         });
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
index b5f744ddc23..b69d1218546 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
@@ -94,7 +94,7 @@ public class RecoverySourceHandlerTests extends ESTestCase {
                     @Override
                     public void close() throws IOException {
                         super.close();
-                        store.directory().sync(Collections.singleton(md.name())); // sync otherwise MDW will mess with it
+                        targetStore.directory().sync(Collections.singleton(md.name())); // sync otherwise MDW will mess with it
                     }
                 };
             } catch (IOException e) {
diff --git a/core/src/test/java/org/elasticsearch/indices/settings/UpdateNumberOfReplicasIT.java b/core/src/test/java/org/elasticsearch/indices/settings/UpdateNumberOfReplicasIT.java
index b257e3bcd5e..1a2f7e4ba18 100644
--- a/core/src/test/java/org/elasticsearch/indices/settings/UpdateNumberOfReplicasIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/settings/UpdateNumberOfReplicasIT.java
@@ -48,7 +48,7 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase {
         assertAcked(prepareCreate("test", 2));
         logger.info("Running Cluster Health");
         ClusterHealthResponse clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet();
-        logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
+        logger.info("Done Cluster Health, status {}", clusterHealth.getStatus());
 
         NumShards numShards = getNumShards("test");
 
@@ -75,7 +75,7 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase {
         assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(settingsBuilder().put("index.number_of_replicas", 2)).execute().actionGet());
         logger.info("Running Cluster Health");
         clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().setWaitForActiveShards(numShards.numPrimaries * 2).execute().actionGet();
-        logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
+        logger.info("Done Cluster Health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));
         assertThat(clusterHealth.getIndices().get("test").getActivePrimaryShards(), equalTo(numShards.numPrimaries));
@@ -88,7 +88,7 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase {
 
         logger.info("Running Cluster Health");
         clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForRelocatingShards(0).setWaitForNodes(">=3").execute().actionGet();
-        logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
+        logger.info("Done Cluster Health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
         assertThat(clusterHealth.getIndices().get("test").getActivePrimaryShards(), equalTo(numShards.numPrimaries));
@@ -106,7 +106,7 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase {
 
         logger.info("Running Cluster Health");
         clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForRelocatingShards(0).setWaitForNodes(">=3").execute().actionGet();
-        logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
+        logger.info("Done Cluster Health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
         assertThat(clusterHealth.getIndices().get("test").getActivePrimaryShards(), equalTo(numShards.numPrimaries));
@@ -128,7 +128,7 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase {
 
         logger.info("--> running cluster health");
         ClusterHealthResponse clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForActiveShards(numShards.numPrimaries * 2).execute().actionGet();
-        logger.info("--> done cluster health, status " + clusterHealth.getStatus());
+        logger.info("--> done cluster health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
         assertThat(clusterHealth.getIndices().get("test").getActivePrimaryShards(), equalTo(numShards.numPrimaries));
@@ -140,7 +140,7 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase {
 
         logger.info("--> running cluster health");
         clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForActiveShards(numShards.numPrimaries * 3).setWaitForNodes(">=3").execute().actionGet();
-        logger.info("--> done cluster health, status " + clusterHealth.getStatus());
+        logger.info("--> done cluster health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
         assertThat(clusterHealth.getIndices().get("test").getActivePrimaryShards(), equalTo(numShards.numPrimaries));
@@ -153,7 +153,7 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase {
 
         logger.info("--> running cluster health");
         clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForActiveShards(numShards.numPrimaries * 2).setWaitForNodes(">=2").execute().actionGet();
-        logger.info("--> done cluster health, status " + clusterHealth.getStatus());
+        logger.info("--> done cluster health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
         assertThat(clusterHealth.getIndices().get("test").getActivePrimaryShards(), equalTo(numShards.numPrimaries));
@@ -166,7 +166,7 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase {
 
         logger.info("--> running cluster health");
         clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes(">=1").setWaitForActiveShards(numShards.numPrimaries).execute().actionGet();
-        logger.info("--> done cluster health, status " + clusterHealth.getStatus());
+        logger.info("--> done cluster health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
         assertThat(clusterHealth.getIndices().get("test").getActivePrimaryShards(), equalTo(numShards.numPrimaries));
@@ -183,7 +183,7 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase {
 
         logger.info("--> running cluster health");
         ClusterHealthResponse clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForActiveShards(numShards.numPrimaries * 2).execute().actionGet();
-        logger.info("--> done cluster health, status " + clusterHealth.getStatus());
+        logger.info("--> done cluster health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
         assertThat(clusterHealth.getIndices().get("test").getActivePrimaryShards(), equalTo(numShards.numPrimaries));
@@ -195,7 +195,7 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase {
 
         logger.info("--> running cluster health");
         clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForActiveShards(numShards.numPrimaries * 3).execute().actionGet();
-        logger.info("--> done cluster health, status " + clusterHealth.getStatus());
+        logger.info("--> done cluster health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
         assertThat(clusterHealth.getIndices().get("test").getActivePrimaryShards(), equalTo(numShards.numPrimaries));
@@ -208,7 +208,7 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase {
 
         logger.info("--> running cluster health");
         clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForNodes(">=2").setWaitForActiveShards(numShards.numPrimaries * 2).execute().actionGet();
-        logger.info("--> done cluster health, status " + clusterHealth.getStatus());
+        logger.info("--> done cluster health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
         assertThat(clusterHealth.getIndices().get("test").getActivePrimaryShards(), equalTo(numShards.numPrimaries));
@@ -221,7 +221,7 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase {
 
         logger.info("--> running cluster health");
         clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().setWaitForNodes(">=1").setWaitForActiveShards(numShards.numPrimaries).execute().actionGet();
-        logger.info("--> done cluster health, status " + clusterHealth.getStatus());
+        logger.info("--> done cluster health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));
         assertThat(clusterHealth.getIndices().get("test").getActivePrimaryShards(), equalTo(numShards.numPrimaries));
@@ -237,7 +237,7 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase {
 
         logger.info("--> running cluster health");
         ClusterHealthResponse clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForActiveShards(numShards.numPrimaries * 3).execute().actionGet();
-        logger.info("--> done cluster health, status " + clusterHealth.getStatus());
+        logger.info("--> done cluster health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
         assertThat(clusterHealth.getIndices().get("test").getActivePrimaryShards(), equalTo(numShards.numPrimaries));
@@ -253,7 +253,7 @@ public class UpdateNumberOfReplicasIT extends ESIntegTestCase {
 
         logger.info("--> running cluster health");
         clusterHealth = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().setWaitForActiveShards(numShards.numPrimaries * 4).execute().actionGet();
-        logger.info("--> done cluster health, status " + clusterHealth.getStatus());
+        logger.info("--> done cluster health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
         assertThat(clusterHealth.getIndices().get("test").getActivePrimaryShards(), equalTo(numShards.numPrimaries));
diff --git a/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java b/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
index 78d5e2203f5..642d646fe9e 100644
--- a/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
@@ -65,7 +65,7 @@ public class UpdateSettingsIT extends ESIntegTestCase {
         IndexMetaData indexMetaData = client().admin().cluster().prepareState().execute().actionGet().getState().metaData().index("test");
         assertEquals(indexMetaData.getSettings().get("index.refresh_interval"), "-1");
         for (IndicesService service : internalCluster().getInstances(IndicesService.class)) {
-            IndexService indexService = service.indexService("test");
+            IndexService indexService = service.indexService(resolveIndex("test"));
             if (indexService != null) {
                 assertEquals(indexService.getIndexSettings().getRefreshInterval().millis(), -1);
                 assertEquals(indexService.getIndexSettings().getFlushThresholdSize().bytes(), 1024);
@@ -79,7 +79,7 @@ public class UpdateSettingsIT extends ESIntegTestCase {
         indexMetaData = client().admin().cluster().prepareState().execute().actionGet().getState().metaData().index("test");
         assertNull(indexMetaData.getSettings().get("index.refresh_interval"));
         for (IndicesService service : internalCluster().getInstances(IndicesService.class)) {
-            IndexService indexService = service.indexService("test");
+            IndexService indexService = service.indexService(resolveIndex("test"));
             if (indexService != null) {
                 assertEquals(indexService.getIndexSettings().getRefreshInterval().millis(), 1000);
                 assertEquals(indexService.getIndexSettings().getFlushThresholdSize().bytes(), 1024);
@@ -137,7 +137,8 @@ public class UpdateSettingsIT extends ESIntegTestCase {
                     .execute().actionGet();
             fail("can't change number of replicas on a closed index");
         } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Can't update [index.number_of_replicas] on closed indices [[test]] - can leave index in an unopenable state");
+            assertTrue(ex.getMessage(), ex.getMessage().startsWith("Can't update [index.number_of_replicas] on closed indices [[test/"));
+            assertTrue(ex.getMessage(), ex.getMessage().endsWith("]] - can leave index in an unopenable state"));
             // expected
         }
         client().admin().indices().prepareUpdateSettings("test")
diff --git a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
index 8a9fa191854..49819e1180a 100644
--- a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.cluster.ClusterInfo;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlocks;
@@ -40,6 +39,7 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
@@ -47,6 +47,7 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.gateway.GatewayAllocator;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperService;
@@ -376,12 +377,13 @@ public class RareClusterStateIT extends ESIntegTestCase {
                 putMappingResponse.set(e);
             }
         });
+        final Index index = resolveIndex("index");
         // Wait for mappings to be available on master
         assertBusy(new Runnable() {
             @Override
             public void run() {
                 final IndicesService indicesService = internalCluster().getInstance(IndicesService.class, master);
-                final IndexService indexService = indicesService.indexServiceSafe("index");
+                final IndexService indexService = indicesService.indexServiceSafe(index);
                 assertNotNull(indexService);
                 final MapperService mapperService = indexService.mapperService();
                 DocumentMapper mapper = mapperService.documentMapper("type");
diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
index 467aa4d3309..12e6d2b4799 100644
--- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
@@ -58,8 +58,10 @@ import java.io.IOException;
 import java.util.EnumSet;
 import java.util.Random;
 
+import static org.elasticsearch.cluster.metadata.IndexMetaData.PROTO;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
+import static org.elasticsearch.common.xcontent.XContentFactory.contentBuilder;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
@@ -79,13 +81,23 @@ public class IndexStatsIT extends ESIntegTestCase {
         //Filter/Query cache is cleaned periodically, default is 60s, so make sure it runs often. Thread.sleep for 60s is bad
         return Settings.settingsBuilder().put(super.nodeSettings(nodeOrdinal))
                 .put(IndicesService.INDICES_CACHE_CLEAN_INTERVAL_SETTING.getKey(), "1ms")
-                .put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)
-                .put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), IndexModule.INDEX_QUERY_CACHE)
                 .build();
     }
 
+    @Override
+    public Settings indexSettings() {
+        return Settings.settingsBuilder().put(super.indexSettings())
+            .put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)
+            .put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), IndexModule.INDEX_QUERY_CACHE)
+            .build();
+    }
+
+    private Settings.Builder settingsBuilder() {
+        return Settings.builder().put(indexSettings());
+    }
+
     public void testFieldDataStats() {
-        client().admin().indices().prepareCreate("test").setSettings(Settings.settingsBuilder().put("index.number_of_shards", 2)).execute().actionGet();
+        client().admin().indices().prepareCreate("test").setSettings(settingsBuilder().put("index.number_of_shards", 2)).execute().actionGet();
         ensureGreen();
         client().prepareIndex("test", "type", "1").setSource("field", "value1", "field2", "value1").execute().actionGet();
         client().prepareIndex("test", "type", "2").setSource("field", "value2", "field2", "value2").execute().actionGet();
@@ -130,7 +142,7 @@ public class IndexStatsIT extends ESIntegTestCase {
 
     public void testClearAllCaches() throws Exception {
         client().admin().indices().prepareCreate("test")
-                .setSettings(Settings.settingsBuilder().put("index.number_of_replicas", 0).put("index.number_of_shards", 2))
+                .setSettings(settingsBuilder().put("index.number_of_replicas", 0).put("index.number_of_shards", 2))
                 .execute().actionGet();
         ensureGreen();
         client().admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
@@ -276,7 +288,7 @@ public class IndexStatsIT extends ESIntegTestCase {
 
     public void testNonThrottleStats() throws Exception {
         assertAcked(prepareCreate("test")
-                .setSettings(Settings.builder()
+                .setSettings(settingsBuilder()
                                 .put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "merge")
                                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
                                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
@@ -308,7 +320,7 @@ public class IndexStatsIT extends ESIntegTestCase {
 
     public void testThrottleStats() throws Exception {
         assertAcked(prepareCreate("test")
-                    .setSettings(Settings.builder()
+                    .setSettings(settingsBuilder()
                                  .put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "merge")
                                  .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
                                  .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
@@ -564,6 +576,10 @@ public class IndexStatsIT extends ESIntegTestCase {
 
         IndicesStatsResponse stats = builder.execute().actionGet();
         for (Flag flag : values) {
+            if (flag == Flag.Suggest) {
+                // suggest flag is unused
+                continue;
+            }
             assertThat(isSet(flag, stats.getPrimaries()), equalTo(false));
             assertThat(isSet(flag, stats.getTotal()), equalTo(false));
         }
@@ -599,6 +615,10 @@ public class IndexStatsIT extends ESIntegTestCase {
         }
 
         for (Flag flag : EnumSet.complementOf(flags)) { // check the complement
+            if (flag == Flag.Suggest) {
+                // suggest flag is unused
+                continue;
+            }
             assertThat(isSet(flag, stats.getPrimaries()), equalTo(false));
             assertThat(isSet(flag, stats.getTotal()), equalTo(false));
         }
@@ -650,7 +670,7 @@ public class IndexStatsIT extends ESIntegTestCase {
 
     public void testFlagOrdinalOrder() {
         Flag[] flags = new Flag[]{Flag.Store, Flag.Indexing, Flag.Get, Flag.Search, Flag.Merge, Flag.Flush, Flag.Refresh,
-                Flag.QueryCache, Flag.FieldData, Flag.Docs, Flag.Warmer, Flag.Percolate, Flag.Completion, Flag.Segments,
+                Flag.QueryCache, Flag.FieldData, Flag.Docs, Flag.Warmer, Flag.PercolatorCache, Flag.Completion, Flag.Segments,
                 Flag.Translog, Flag.Suggest, Flag.RequestCache, Flag.Recovery};
 
         assertThat(flags.length, equalTo(Flag.values().length));
@@ -891,7 +911,7 @@ public class IndexStatsIT extends ESIntegTestCase {
             case Warmer:
                 builder.setWarmer(set);
                 break;
-            case Percolate:
+            case PercolatorCache:
                 builder.setPercolate(set);
                 break;
             case Completion:
@@ -903,8 +923,7 @@ public class IndexStatsIT extends ESIntegTestCase {
             case Translog:
                 builder.setTranslog(set);
                 break;
-            case Suggest:
-                builder.setSuggest(set);
+            case Suggest: // unused
                 break;
             case RequestCache:
                 builder.setRequestCache(set);
@@ -942,16 +961,16 @@ public class IndexStatsIT extends ESIntegTestCase {
                 return response.getStore() != null;
             case Warmer:
                 return response.getWarmer() != null;
-            case Percolate:
-                return response.getPercolate() != null;
+            case PercolatorCache:
+                return response.getPercolatorCache() != null;
             case Completion:
                 return response.getCompletion() != null;
             case Segments:
                 return response.getSegments() != null;
             case Translog:
                 return response.getTranslog() != null;
-            case Suggest:
-                return response.getSuggest() != null;
+            case Suggest: // unused
+                return true;
             case RequestCache:
                 return response.getRequestCache() != null;
             case Recovery:
@@ -988,7 +1007,7 @@ public class IndexStatsIT extends ESIntegTestCase {
     }
 
     public void testFilterCacheStats() throws Exception {
-        assertAcked(prepareCreate("index").setSettings("number_of_replicas", 0).get());
+        assertAcked(prepareCreate("index").setSettings(Settings.builder().put(indexSettings()).put("number_of_replicas", 0).build()).get());
         indexRandom(true,
                 client().prepareIndex("index", "type", "1").setSource("foo", "bar"),
                 client().prepareIndex("index", "type", "2").setSource("foo", "baz"));
diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
index 948c005bf33..84d86324fee 100644
--- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
@@ -21,7 +21,6 @@ package org.elasticsearch.indices.store;
 
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
@@ -36,6 +35,7 @@ import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Settings;
@@ -112,12 +112,14 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
                                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1))
         );
         ensureGreen("test");
+        ClusterState state = client().admin().cluster().prepareState().get().getState();
+        Index index = state.metaData().index("test").getIndex();
 
         logger.info("--> making sure that shard and its replica are allocated on node_1 and node_2");
-        assertThat(Files.exists(shardDirectory(node_1, "test", 0)), equalTo(true));
-        assertThat(Files.exists(indexDirectory(node_1, "test")), equalTo(true));
-        assertThat(Files.exists(shardDirectory(node_2, "test", 0)), equalTo(true));
-        assertThat(Files.exists(indexDirectory(node_2, "test")), equalTo(true));
+        assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
+        assertThat(Files.exists(indexDirectory(node_1, index)), equalTo(true));
+        assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(true));
+        assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(true));
 
         logger.info("--> starting node server3");
         final String node_3 = internalCluster().startNode(Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false));
@@ -128,12 +130,12 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
                 .get();
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
 
-        assertThat(Files.exists(shardDirectory(node_1, "test", 0)), equalTo(true));
-        assertThat(Files.exists(indexDirectory(node_1, "test")), equalTo(true));
-        assertThat(Files.exists(shardDirectory(node_2, "test", 0)), equalTo(true));
-        assertThat(Files.exists(indexDirectory(node_2, "test")), equalTo(true));
-        assertThat(Files.exists(shardDirectory(node_3, "test", 0)), equalTo(false));
-        assertThat(Files.exists(indexDirectory(node_3, "test")), equalTo(false));
+        assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
+        assertThat(Files.exists(indexDirectory(node_1, index)), equalTo(true));
+        assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(true));
+        assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(true));
+        assertThat(Files.exists(shardDirectory(node_3, index, 0)), equalTo(false));
+        assertThat(Files.exists(indexDirectory(node_3, index)), equalTo(false));
 
         logger.info("--> move shard from node_1 to node_3, and wait for relocation to finish");
 
@@ -161,12 +163,12 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
                 .get();
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
 
-        assertThat(waitForShardDeletion(node_1, "test", 0), equalTo(false));
-        assertThat(waitForIndexDeletion(node_1, "test"), equalTo(false));
-        assertThat(Files.exists(shardDirectory(node_2, "test", 0)), equalTo(true));
-        assertThat(Files.exists(indexDirectory(node_2, "test")), equalTo(true));
-        assertThat(Files.exists(shardDirectory(node_3, "test", 0)), equalTo(true));
-        assertThat(Files.exists(indexDirectory(node_3, "test")), equalTo(true));
+        assertThat(waitForShardDeletion(node_1, index, 0), equalTo(false));
+        assertThat(waitForIndexDeletion(node_1, index), equalTo(false));
+        assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(true));
+        assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(true));
+        assertThat(Files.exists(shardDirectory(node_3, index, 0)), equalTo(true));
+        assertThat(Files.exists(indexDirectory(node_3, index)), equalTo(true));
 
     }
 
@@ -180,16 +182,18 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
                                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
         );
         ensureGreen("test");
-        assertThat(Files.exists(shardDirectory(node_1, "test", 0)), equalTo(true));
-        assertThat(Files.exists(indexDirectory(node_1, "test")), equalTo(true));
+        ClusterState state = client().admin().cluster().prepareState().get().getState();
+        Index index = state.metaData().index("test").getIndex();
+        assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
+        assertThat(Files.exists(indexDirectory(node_1, index)), equalTo(true));
 
         final String node_2 = internalCluster().startDataOnlyNode(Settings.builder().build());
         assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());
 
-        assertThat(Files.exists(shardDirectory(node_1, "test", 0)), equalTo(true));
-        assertThat(Files.exists(indexDirectory(node_1, "test")), equalTo(true));
-        assertThat(Files.exists(shardDirectory(node_2, "test", 0)), equalTo(false));
-        assertThat(Files.exists(indexDirectory(node_2, "test")), equalTo(false));
+        assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
+        assertThat(Files.exists(indexDirectory(node_1, index)), equalTo(true));
+        assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(false));
+        assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(false));
 
         // add a transport delegate that will prevent the shard active request to succeed the first time after relocation has finished.
         // node_1 will then wait for the next cluster state change before it tries a next attempt to delete the shard.
@@ -220,14 +224,14 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
         // it must still delete the shard, even if it cannot find it anymore in indicesservice
         client().admin().indices().prepareDelete("test").get();
 
-        assertThat(waitForShardDeletion(node_1, "test", 0), equalTo(false));
-        assertThat(waitForIndexDeletion(node_1, "test"), equalTo(false));
-        assertThat(Files.exists(shardDirectory(node_1, "test", 0)), equalTo(false));
-        assertThat(Files.exists(indexDirectory(node_1, "test")), equalTo(false));
-        assertThat(waitForShardDeletion(node_2, "test", 0), equalTo(false));
-        assertThat(waitForIndexDeletion(node_2, "test"), equalTo(false));
-        assertThat(Files.exists(shardDirectory(node_2, "test", 0)), equalTo(false));
-        assertThat(Files.exists(indexDirectory(node_2, "test")), equalTo(false));
+        assertThat(waitForShardDeletion(node_1, index, 0), equalTo(false));
+        assertThat(waitForIndexDeletion(node_1, index), equalTo(false));
+        assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(false));
+        assertThat(Files.exists(indexDirectory(node_1, index)), equalTo(false));
+        assertThat(waitForShardDeletion(node_2, index, 0), equalTo(false));
+        assertThat(waitForIndexDeletion(node_2, index), equalTo(false));
+        assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(false));
+        assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(false));
     }
 
     public void testShardsCleanup() throws Exception {
@@ -241,9 +245,11 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
         );
         ensureGreen("test");
 
+        ClusterState state = client().admin().cluster().prepareState().get().getState();
+        Index index = state.metaData().index("test").getIndex();
         logger.info("--> making sure that shard and its replica are allocated on node_1 and node_2");
-        assertThat(Files.exists(shardDirectory(node_1, "test", 0)), equalTo(true));
-        assertThat(Files.exists(shardDirectory(node_2, "test", 0)), equalTo(true));
+        assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
+        assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(true));
 
         logger.info("--> starting node server3");
         String node_3 = internalCluster().startNode();
@@ -255,9 +261,9 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
 
         logger.info("--> making sure that shard is not allocated on server3");
-        assertThat(waitForShardDeletion(node_3, "test", 0), equalTo(false));
+        assertThat(waitForShardDeletion(node_3, index, 0), equalTo(false));
 
-        Path server2Shard = shardDirectory(node_2, "test", 0);
+        Path server2Shard = shardDirectory(node_2, index, 0);
         logger.info("--> stopping node " + node_2);
         internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node_2));
 
@@ -268,14 +274,14 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
                 .setWaitForRelocatingShards(0)
                 .get();
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
-        logger.info("--> done cluster_health, status " + clusterHealth.getStatus());
+        logger.info("--> done cluster_health, status {}", clusterHealth.getStatus());
 
         assertThat(Files.exists(server2Shard), equalTo(true));
 
         logger.info("--> making sure that shard and its replica exist on server1, server2 and server3");
-        assertThat(Files.exists(shardDirectory(node_1, "test", 0)), equalTo(true));
+        assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
         assertThat(Files.exists(server2Shard), equalTo(true));
-        assertThat(Files.exists(shardDirectory(node_3, "test", 0)), equalTo(true));
+        assertThat(Files.exists(shardDirectory(node_3, index, 0)), equalTo(true));
 
         logger.info("--> starting node node_4");
         final String node_4 = internalCluster().startNode();
@@ -284,9 +290,9 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
         ensureGreen();
 
         logger.info("--> making sure that shard and its replica are allocated on server1 and server3 but not on server2");
-        assertThat(Files.exists(shardDirectory(node_1, "test", 0)), equalTo(true));
-        assertThat(Files.exists(shardDirectory(node_3, "test", 0)), equalTo(true));
-        assertThat(waitForShardDeletion(node_4, "test", 0), equalTo(false));
+        assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
+        assertThat(Files.exists(shardDirectory(node_3, index, 0)), equalTo(true));
+        assertThat(waitForShardDeletion(node_4, index, 0), equalTo(false));
     }
 
     public void testShardActiveElsewhereDoesNotDeleteAnother() throws Exception {
@@ -303,6 +309,7 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
         final String node4 = nodesFutures.get().get(3);
 
         assertAcked(prepareCreate("test").setSettings(Settings.builder()
+                        .put(indexSettings())
                         .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 3)
                         .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
                         .put(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + "_name", node4)
@@ -336,10 +343,11 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
         // we have to do this in two steps as we now do async shard fetching before assigning, so the change to the
         // allocation filtering may not have immediate effect
         // TODO: we should add an easier to do this. It's too much of a song and dance..
+        Index index = resolveIndex("test");
         assertBusy(new Runnable() {
             @Override
             public void run() {
-                assertTrue(internalCluster().getInstance(IndicesService.class, node4).hasIndex("test"));
+                assertTrue(internalCluster().getInstance(IndicesService.class, node4).hasIndex(index));
             }
         });
 
@@ -425,30 +433,30 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
         waitNoPendingTasksOnAll();
         logger.info("Checking if shards aren't removed");
         for (int shard : node2Shards) {
-            assertTrue(waitForShardDeletion(nonMasterNode, "test", shard));
+            assertTrue(waitForShardDeletion(nonMasterNode, index, shard));
         }
     }
 
-    private Path indexDirectory(String server, String index) {
+    private Path indexDirectory(String server, Index index) {
         NodeEnvironment env = internalCluster().getInstance(NodeEnvironment.class, server);
         final Path[] paths = env.indexPaths(index);
         assert paths.length == 1;
         return paths[0];
     }
 
-    private Path shardDirectory(String server, String index, int shard) {
+    private Path shardDirectory(String server, Index index, int shard) {
         NodeEnvironment env = internalCluster().getInstance(NodeEnvironment.class, server);
-        final Path[] paths = env.availableShardPaths(new ShardId(index, "_na_", shard));
+        final Path[] paths = env.availableShardPaths(new ShardId(index, shard));
         assert paths.length == 1;
         return paths[0];
     }
 
-    private boolean waitForShardDeletion(final String server, final String index, final int shard) throws InterruptedException {
+    private boolean waitForShardDeletion(final String server, final Index index, final int shard) throws InterruptedException {
         awaitBusy(() -> !Files.exists(shardDirectory(server, index, shard)));
         return Files.exists(shardDirectory(server, index, shard));
     }
 
-    private boolean waitForIndexDeletion(final String server, final String index) throws InterruptedException {
+    private boolean waitForIndexDeletion(final String server, final Index index) throws InterruptedException {
         awaitBusy(() -> !Files.exists(indexDirectory(server, index)));
         return Files.exists(indexDirectory(server, index));
     }
diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
index e909af62668..badcbde193b 100644
--- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
@@ -30,19 +30,25 @@ import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.cluster.TestClusterService;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
+import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Before;
+import org.junit.BeforeClass;
 
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.Version.CURRENT;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.VersionUtils.randomVersion;
 
 /**
@@ -57,13 +63,35 @@ public class IndicesStoreTests extends ESTestCase {
         NOT_STARTED_STATES = set.toArray(new ShardRoutingState[set.size()]);
     }
 
+    private static ThreadPool threadPool;
+
     private IndicesStore indicesStore;
     private DiscoveryNode localNode;
 
+    private ClusterService clusterService;
+
+    @BeforeClass
+    public static void beforeClass() {
+        threadPool = new ThreadPool("ShardReplicationTests");
+    }
+
+    @AfterClass
+    public static void afterClass() {
+        ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
+        threadPool = null;
+    }
+
     @Before
     public void before() {
         localNode = new DiscoveryNode("abc", new LocalTransportAddress("abc"), Version.CURRENT);
-        indicesStore = new IndicesStore(Settings.EMPTY, null, new TestClusterService(), new TransportService(null, null), null);
+        clusterService = createClusterService(threadPool);
+        indicesStore = new IndicesStore(Settings.EMPTY, null, clusterService, new TransportService(null, null), null);
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        clusterService.close();
     }
 
     public void testShardCanBeDeletedNoShardRouting() throws Exception {
@@ -87,7 +115,7 @@ public class IndicesStoreTests extends ESTestCase {
 
         for (int i = 0; i < numShards; i++) {
             int unStartedShard = randomInt(numReplicas);
-            for (int j=0; j <= numReplicas; j++) {
+            for (int j = 0; j <= numReplicas; j++) {
                 ShardRoutingState state;
                 if (j == unStartedShard) {
                     state = randomFrom(NOT_STARTED_STATES);
diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
index cce687fcec3..d14a411c332 100644
--- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
@@ -131,7 +131,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
                 .addField("field1").addField("field2")
                 .execute().actionGet();
         if (searchResponse.getFailedShards() > 0) {
-            logger.warn("failed search " + Arrays.toString(searchResponse.getShardFailures()));
+            logger.warn("failed search {}", Arrays.toString(searchResponse.getShardFailures()));
         }
         assertHitCount(searchResponse, 1);
         assertThat(searchResponse.getHits().getAt(0).field("field1").value().toString(), equalTo("value1"));
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
index abfe18f8c58..a415b0992a7 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
@@ -67,9 +67,11 @@ public class IngestProcessorNotInstalledOnAllNodesIT extends ESIntegTestCase {
 
     public void testFailPipelineCreation() throws Exception {
         installPlugin = true;
-        internalCluster().startNode();
+        String node1 = internalCluster().startNode();
         installPlugin = false;
-        internalCluster().startNode();
+        String node2 = internalCluster().startNode();
+        ensureStableCluster(2, node1);
+        ensureStableCluster(2, node2);
 
         try {
             client().admin().cluster().preparePutPipeline("_id", pipelineSource).get();
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java b/core/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java
new file mode 100644
index 00000000000..e7064b7e449
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+public class IngestStatsTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        IngestStats.Stats total = new IngestStats.Stats(5, 10, 20, 30);
+        IngestStats.Stats foo = new IngestStats.Stats(50, 100, 200, 300);
+        IngestStats ingestStats = new IngestStats(total, Collections.singletonMap("foo", foo));
+        IngestStats serialize = serialize(ingestStats);
+        assertNotSame(serialize, ingestStats);
+        assertNotSame(serialize.getTotalStats(), total);
+        assertEquals(total.getIngestCount(), serialize.getTotalStats().getIngestCount());
+        assertEquals(total.getIngestFailedCount(), serialize.getTotalStats().getIngestFailedCount());
+        assertEquals(total.getIngestTimeInMillis(), serialize.getTotalStats().getIngestTimeInMillis());
+        assertEquals(total.getIngestCurrent(), serialize.getTotalStats().getIngestCurrent());
+
+        assertEquals(ingestStats.getStatsPerPipeline().size(), 1);
+        assertTrue(ingestStats.getStatsPerPipeline().containsKey("foo"));
+
+        Map<String, IngestStats.Stats> left = ingestStats.getStatsPerPipeline();
+        Map<String, IngestStats.Stats> right = serialize.getStatsPerPipeline();
+
+        assertEquals(right.size(), 1);
+        assertTrue(right.containsKey("foo"));
+        assertEquals(left.size(), 1);
+        assertTrue(left.containsKey("foo"));
+        IngestStats.Stats leftStats = left.get("foo");
+        IngestStats.Stats rightStats = right.get("foo");
+        assertEquals(leftStats.getIngestCount(), rightStats.getIngestCount());
+        assertEquals(leftStats.getIngestFailedCount(), rightStats.getIngestFailedCount());
+        assertEquals(leftStats.getIngestTimeInMillis(), rightStats.getIngestTimeInMillis());
+        assertEquals(leftStats.getIngestCurrent(), rightStats.getIngestCurrent());
+    }
+
+    private <T> T serialize(Writeable<T> writeable) throws IOException {
+        BytesStreamOutput out = new BytesStreamOutput();
+        writeable.writeTo(out);
+        StreamInput in = StreamInput.wrap(out.bytes());
+        return writeable.readFrom(in);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
index e644df2a83a..b84ba928be4 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.ingest.core.CompoundProcessor;
 import org.elasticsearch.ingest.core.IngestDocument;
@@ -38,15 +39,16 @@ import org.mockito.ArgumentMatcher;
 import org.mockito.invocation.InvocationOnMock;
 
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Matchers.anyBoolean;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Matchers.argThat;
 import static org.mockito.Mockito.doAnswer;
@@ -341,6 +343,43 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         verify(completionHandler, times(1)).accept(null);
     }
 
+    public void testStats() throws Exception {
+        IngestStats ingestStats = executionService.stats();
+        assertThat(ingestStats.getStatsPerPipeline().size(), equalTo(0));
+        assertThat(ingestStats.getTotalStats().getIngestCount(), equalTo(0L));
+        assertThat(ingestStats.getTotalStats().getIngestCurrent(), equalTo(0L));
+        assertThat(ingestStats.getTotalStats().getIngestFailedCount(), equalTo(0L));
+        assertThat(ingestStats.getTotalStats().getIngestTimeInMillis(), equalTo(0L));
+
+        when(store.get("_id1")).thenReturn(new Pipeline("_id1", null, new CompoundProcessor()));
+        when(store.get("_id2")).thenReturn(new Pipeline("_id2", null, new CompoundProcessor()));
+
+        Map<String, PipelineConfiguration> configurationMap = new HashMap<>();
+        configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}")));
+        configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}")));
+        executionService.updatePipelineStats(new IngestMetadata(configurationMap));
+
+        Consumer<Throwable> failureHandler = mock(Consumer.class);
+        Consumer<Boolean> completionHandler = mock(Consumer.class);
+
+        IndexRequest indexRequest = new IndexRequest("_index");
+        indexRequest.setPipeline("_id1");
+        executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler);
+        ingestStats = executionService.stats();
+        assertThat(ingestStats.getStatsPerPipeline().size(), equalTo(2));
+        assertThat(ingestStats.getStatsPerPipeline().get("_id1").getIngestCount(), equalTo(1L));
+        assertThat(ingestStats.getStatsPerPipeline().get("_id2").getIngestCount(), equalTo(0L));
+        assertThat(ingestStats.getTotalStats().getIngestCount(), equalTo(1L));
+
+        indexRequest.setPipeline("_id2");
+        executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler);
+        ingestStats = executionService.stats();
+        assertThat(ingestStats.getStatsPerPipeline().size(), equalTo(2));
+        assertThat(ingestStats.getStatsPerPipeline().get("_id1").getIngestCount(), equalTo(1L));
+        assertThat(ingestStats.getStatsPerPipeline().get("_id2").getIngestCount(), equalTo(1L));
+        assertThat(ingestStats.getTotalStats().getIngestCount(), equalTo(2L));
+    }
+
     private IngestDocument eqID(String index, String type, String id, Map<String, Object> source) {
         return argThat(new IngestDocumentMatcher(index, type, id, source));
     }
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/ForEachProcessorTests.java b/core/src/test/java/org/elasticsearch/ingest/processor/ForEachProcessorTests.java
index 2ef911e270c..9f76a32e0c2 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/ForEachProcessorTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/processor/ForEachProcessorTests.java
@@ -20,9 +20,12 @@
 package org.elasticsearch.ingest.processor;
 
 import org.elasticsearch.ingest.TestProcessor;
+import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.ingest.core.CompoundProcessor;
 import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.core.TemplateService;
+import org.elasticsearch.ingest.core.ValueSource;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -120,6 +123,42 @@ public class ForEachProcessorTests extends ESTestCase {
         assertThat(ingestDocument.getFieldValue("values.1.id", String.class), equalTo("_id"));
     }
 
+    public void testRestOfTheDocumentIsAvailable() throws Exception {
+        List<Map<String, Object>> values = new ArrayList<>();
+        for (int i = 0; i < 5; i++) {
+            Map<String, Object> object = new HashMap<>();
+            object.put("field", "value");
+            values.add(object);
+        }
+        Map<String, Object> document = new HashMap<>();
+        document.put("values", values);
+        document.put("flat_values", new ArrayList<>());
+        document.put("other", "value");
+        IngestDocument ingestDocument = new IngestDocument("_index", "_type", "_id", null, null, null, null, document);
+
+        TemplateService ts = TestTemplateService.instance();
+        ForEachProcessor processor = new ForEachProcessor(
+                "_tag", "values", Arrays.asList(
+                new AppendProcessor("_tag", ts.compile("flat_values"), ValueSource.wrap("value", ts)),
+                new SetProcessor("_tag", ts.compile("_value.new_field"), (model) -> model.get("other")))
+        );
+        processor.execute(ingestDocument);
+
+        assertThat(ingestDocument.getFieldValue("values.0.new_field", String.class), equalTo("value"));
+        assertThat(ingestDocument.getFieldValue("values.1.new_field", String.class), equalTo("value"));
+        assertThat(ingestDocument.getFieldValue("values.2.new_field", String.class), equalTo("value"));
+        assertThat(ingestDocument.getFieldValue("values.3.new_field", String.class), equalTo("value"));
+        assertThat(ingestDocument.getFieldValue("values.4.new_field", String.class), equalTo("value"));
+
+        List<String> flatValues = ingestDocument.getFieldValue("flat_values", List.class);
+        assertThat(flatValues.size(), equalTo(5));
+        assertThat(flatValues.get(0), equalTo("value"));
+        assertThat(flatValues.get(1), equalTo("value"));
+        assertThat(flatValues.get(2), equalTo("value"));
+        assertThat(flatValues.get(3), equalTo("value"));
+        assertThat(flatValues.get(4), equalTo("value"));
+    }
+
     public void testRandom() throws Exception {
         int numProcessors = randomInt(8);
         List<Processor> processors = new ArrayList<>(numProcessors);
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java b/core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java
new file mode 100644
index 00000000000..e53eec56cf1
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.processor;
+
+import org.elasticsearch.action.ingest.SimulateProcessorResult;
+import org.elasticsearch.ingest.TestProcessor;
+import org.elasticsearch.ingest.core.CompoundProcessor;
+import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.ingest.core.CompoundProcessor.ON_FAILURE_MESSAGE_FIELD;
+import static org.elasticsearch.ingest.core.CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD;
+import static org.elasticsearch.ingest.core.CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD;
+import static org.elasticsearch.ingest.processor.TrackingResultProcessor.decorate;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public class TrackingResultProcessorTests extends ESTestCase {
+
+    private IngestDocument ingestDocument;
+    private List<SimulateProcessorResult> resultList;
+
+    @Before
+    public void init() {
+        ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
+        resultList = new ArrayList<>();
+    }
+
+    public void testActualProcessor() throws Exception {
+        TestProcessor actualProcessor = new TestProcessor(ingestDocument -> {});
+        TrackingResultProcessor trackingProcessor = new TrackingResultProcessor(actualProcessor, resultList);
+        trackingProcessor.execute(ingestDocument);
+
+        SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(), ingestDocument);
+
+        assertThat(actualProcessor.getInvokedCounter(), equalTo(1));
+        assertThat(resultList.size(), equalTo(1));
+
+        assertThat(resultList.get(0).getIngestDocument(), equalTo(expectedResult.getIngestDocument()));
+        assertThat(resultList.get(0).getFailure(), nullValue());
+        assertThat(resultList.get(0).getProcessorTag(), equalTo(expectedResult.getProcessorTag()));
+    }
+
+    public void testActualCompoundProcessorWithoutOnFailure() throws Exception {
+        RuntimeException exception = new RuntimeException("processor failed");
+        TestProcessor testProcessor = new TestProcessor(ingestDocument -> {  throw exception; });
+        CompoundProcessor actualProcessor = new CompoundProcessor(testProcessor);
+        CompoundProcessor trackingProcessor = decorate(actualProcessor, resultList);
+
+        try {
+            trackingProcessor.execute(ingestDocument);
+        } catch (Exception e) {
+            assertThat(e.getMessage(), equalTo(exception.getMessage()));
+        }
+
+        SimulateProcessorResult expectedFirstResult = new SimulateProcessorResult(testProcessor.getTag(), ingestDocument);
+        assertThat(testProcessor.getInvokedCounter(), equalTo(1));
+        assertThat(resultList.size(), equalTo(1));
+        assertThat(resultList.get(0).getIngestDocument(), nullValue());
+        assertThat(resultList.get(0).getFailure(), equalTo(exception));
+        assertThat(resultList.get(0).getProcessorTag(), equalTo(expectedFirstResult.getProcessorTag()));
+    }
+
+    public void testActualCompoundProcessorWithOnFailure() throws Exception {
+        RuntimeException exception = new RuntimeException("fail");
+        TestProcessor failProcessor = new TestProcessor("fail", "test", ingestDocument -> {  throw exception; });
+        TestProcessor onFailureProcessor = new TestProcessor("success", "test", ingestDocument -> {});
+        CompoundProcessor actualProcessor = new CompoundProcessor(
+            Arrays.asList(new CompoundProcessor(
+                Arrays.asList(failProcessor, onFailureProcessor),
+                Arrays.asList(onFailureProcessor, failProcessor))),
+                Arrays.asList(onFailureProcessor));
+        CompoundProcessor trackingProcessor = decorate(actualProcessor, resultList);
+        trackingProcessor.execute(ingestDocument);
+
+        SimulateProcessorResult expectedFailResult = new SimulateProcessorResult(failProcessor.getTag(), ingestDocument);
+        SimulateProcessorResult expectedSuccessResult = new SimulateProcessorResult(onFailureProcessor.getTag(), ingestDocument);
+
+        assertThat(failProcessor.getInvokedCounter(), equalTo(2));
+        assertThat(onFailureProcessor.getInvokedCounter(), equalTo(2));
+        assertThat(resultList.size(), equalTo(4));
+
+        assertThat(resultList.get(0).getIngestDocument(), nullValue());
+        assertThat(resultList.get(0).getFailure(), equalTo(exception));
+        assertThat(resultList.get(0).getProcessorTag(), equalTo(expectedFailResult.getProcessorTag()));
+
+        Map<String, String> metadata = resultList.get(1).getIngestDocument().getIngestMetadata();
+        assertThat(metadata.get(ON_FAILURE_MESSAGE_FIELD), equalTo("fail"));
+        assertThat(metadata.get(ON_FAILURE_PROCESSOR_TYPE_FIELD), equalTo("test"));
+        assertThat(metadata.get(ON_FAILURE_PROCESSOR_TAG_FIELD), equalTo("fail"));
+        assertThat(resultList.get(1).getFailure(), nullValue());
+        assertThat(resultList.get(1).getProcessorTag(), equalTo(expectedSuccessResult.getProcessorTag()));
+
+        assertThat(resultList.get(2).getIngestDocument(), nullValue());
+        assertThat(resultList.get(2).getFailure(), equalTo(exception));
+        assertThat(resultList.get(2).getProcessorTag(), equalTo(expectedFailResult.getProcessorTag()));
+
+        metadata = resultList.get(3).getIngestDocument().getIngestMetadata();
+        assertThat(metadata.get(ON_FAILURE_MESSAGE_FIELD), equalTo("fail"));
+        assertThat(metadata.get(ON_FAILURE_PROCESSOR_TYPE_FIELD), equalTo("compound"));
+        assertThat(metadata.get(ON_FAILURE_PROCESSOR_TAG_FIELD), equalTo("CompoundProcessor-fail-success-success-fail"));
+        assertThat(resultList.get(3).getFailure(), nullValue());
+        assertThat(resultList.get(3).getProcessorTag(), equalTo(expectedSuccessResult.getProcessorTag()));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
index 33876ef61ad..c979b2f4013 100644
--- a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
+++ b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
@@ -24,7 +24,7 @@ import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
-import org.elasticsearch.common.cli.MockTerminal;
+import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
index 2a845303675..bb56d139111 100644
--- a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
+++ b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
@@ -47,6 +47,7 @@ import org.elasticsearch.transport.TransportInfo;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -137,7 +138,7 @@ public class NodeInfoStreamingTests extends ESTestCase {
         PluginsAndModules plugins = new PluginsAndModules();
         plugins.addModule(DummyPluginInfo.INSTANCE);
         plugins.addPlugin(DummyPluginInfo.INSTANCE);
-        IngestInfo ingestInfo = new IngestInfo();
+        IngestInfo ingestInfo = new IngestInfo(Collections.emptyList());
         return new NodeInfo(VersionUtils.randomVersion(random()), build, node, serviceAttributes, settings, osInfo, process, jvm, threadPoolInfo, transport, htttpInfo, plugins, ingestInfo);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
index e3777e84f9a..03b7a258f4f 100644
--- a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
+++ b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
@@ -21,7 +21,7 @@ package org.elasticsearch.nodesinfo;
 
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
-import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -47,11 +47,11 @@ public class SimpleNodesInfoIT extends ESIntegTestCase {
         final String node_2 = nodesIds.get(1);
 
         ClusterHealthResponse clusterHealth = client().admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForNodes("2").get();
-        logger.info("--> done cluster_health, status " + clusterHealth.getStatus());
+        logger.info("--> done cluster_health, status {}", clusterHealth.getStatus());
 
         String server1NodeId = internalCluster().getInstance(ClusterService.class, node_1).state().nodes().localNodeId();
         String server2NodeId = internalCluster().getInstance(ClusterService.class, node_2).state().nodes().localNodeId();
-        logger.info("--> started nodes: " + server1NodeId + " and " + server2NodeId);
+        logger.info("--> started nodes: {} and {}", server1NodeId, server2NodeId);
 
         NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().execute().actionGet();
         assertThat(response.getNodes().length, is(2));
@@ -91,11 +91,11 @@ public class SimpleNodesInfoIT extends ESIntegTestCase {
         final String node_2 = nodesIds.get(1);
 
         ClusterHealthResponse clusterHealth = client().admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForNodes("2").get();
-        logger.info("--> done cluster_health, status " + clusterHealth.getStatus());
+        logger.info("--> done cluster_health, status {}", clusterHealth.getStatus());
 
         String server1NodeId = internalCluster().getInstance(ClusterService.class, node_1).state().nodes().localNodeId();
         String server2NodeId = internalCluster().getInstance(ClusterService.class, node_2).state().nodes().localNodeId();
-        logger.info("--> started nodes: " + server1NodeId + " and " + server2NodeId);
+        logger.info("--> started nodes: {} and {}", server1NodeId, server2NodeId);
 
         NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().execute().actionGet();
 
diff --git a/core/src/test/java/org/elasticsearch/percolator/ConcurrentPercolatorIT.java b/core/src/test/java/org/elasticsearch/percolator/ConcurrentPercolatorIT.java
deleted file mode 100644
index e99cf51758b..00000000000
--- a/core/src/test/java/org/elasticsearch/percolator/ConcurrentPercolatorIT.java
+++ /dev/null
@@ -1,395 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.action.delete.DeleteResponse;
-import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.percolate.PercolateResponse;
-import org.elasticsearch.action.percolate.PercolateSourceBuilder;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-
-import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.elasticsearch.percolator.PercolatorTestUtil.convertFromTextArray;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.hamcrest.Matchers.arrayContaining;
-import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
-import static org.hamcrest.Matchers.arrayWithSize;
-import static org.hamcrest.Matchers.emptyArray;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.nullValue;
-
-
-/**
- *
- */
-public class ConcurrentPercolatorIT extends ESIntegTestCase {
-    public void testSimpleConcurrentPercolator() throws Exception {
-        // We need to index a document / define mapping, otherwise field1 doesn't get recognized as number field.
-        // If we don't do this, then 'test2' percolate query gets parsed as a TermQuery and not a RangeQuery.
-        // The percolate api doesn't parse the doc if no queries have registered, so it can't lazily create a mapping
-        assertAcked(prepareCreate("index").addMapping("type", "field1", "type=long", "field2", "type=text")); // random # shards better has a mapping!
-        ensureGreen();
-
-        final BytesReference onlyField1 = XContentFactory.jsonBuilder().startObject().startObject("doc")
-                .field("field1", 1)
-                .endObject().endObject().bytes();
-        final BytesReference onlyField2 = XContentFactory.jsonBuilder().startObject().startObject("doc")
-                .field("field2", "value")
-                .endObject().endObject().bytes();
-        final BytesReference bothFields = XContentFactory.jsonBuilder().startObject().startObject("doc")
-                .field("field1", 1)
-                .field("field2", "value")
-                .endObject().endObject().bytes();
-
-        client().prepareIndex("index", "type", "1").setSource(XContentFactory.jsonBuilder().startObject()
-                .field("field1", 1)
-                .field("field2", "value")
-                .endObject()).execute().actionGet();
-
-        client().prepareIndex("index", PercolatorService.TYPE_NAME, "test1")
-                .setSource(XContentFactory.jsonBuilder().startObject().field("query", termQuery("field2", "value")).endObject())
-                .execute().actionGet();
-        client().prepareIndex("index", PercolatorService.TYPE_NAME, "test2")
-                .setSource(XContentFactory.jsonBuilder().startObject().field("query", termQuery("field1", 1)).endObject())
-                .execute().actionGet();
-        refresh(); // make sure it's refreshed
-
-        final CountDownLatch start = new CountDownLatch(1);
-        final AtomicBoolean stop = new AtomicBoolean(false);
-        final AtomicInteger counts = new AtomicInteger(0);
-        final AtomicReference<Throwable> exceptionHolder = new AtomicReference<>();
-        Thread[] threads = new Thread[scaledRandomIntBetween(2, 5)];
-        final int numberOfPercolations = scaledRandomIntBetween(1000, 10000);
-
-        for (int i = 0; i < threads.length; i++) {
-            Runnable r = new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        start.await();
-                        while (!stop.get()) {
-                            int count = counts.incrementAndGet();
-                            if ((count > numberOfPercolations)) {
-                                stop.set(true);
-                            }
-                            PercolateResponse percolate;
-                            if (count % 3 == 0) {
-                                percolate = client().preparePercolate().setIndices("index").setDocumentType("type")
-                                        .setSource(bothFields)
-                                        .execute().actionGet();
-                                assertThat(percolate.getMatches(), arrayWithSize(2));
-                                assertThat(convertFromTextArray(percolate.getMatches(), "index"), arrayContainingInAnyOrder("test1", "test2"));
-                            } else if (count % 3 == 1) {
-                                percolate = client().preparePercolate().setIndices("index").setDocumentType("type")
-                                        .setSource(onlyField2)
-                                        .execute().actionGet();
-                                assertThat(percolate.getMatches(), arrayWithSize(1));
-                                assertThat(convertFromTextArray(percolate.getMatches(), "index"), arrayContaining("test1"));
-                            } else {
-                                percolate = client().preparePercolate().setIndices("index").setDocumentType("type")
-                                        .setSource(onlyField1)
-                                        .execute().actionGet();
-                                assertThat(percolate.getMatches(), arrayWithSize(1));
-                                assertThat(convertFromTextArray(percolate.getMatches(), "index"), arrayContaining("test2"));
-                            }
-                        }
-                    } catch (InterruptedException e) {
-                        Thread.currentThread().interrupt();
-                    } catch (Throwable e) {
-                        exceptionHolder.set(e);
-                        Thread.currentThread().interrupt();
-                    }
-                }
-            };
-            threads[i] = new Thread(r);
-            threads[i].start();
-        }
-
-        start.countDown();
-        for (Thread thread : threads) {
-            thread.join();
-        }
-
-        Throwable assertionError = exceptionHolder.get();
-        if (assertionError != null) {
-            assertionError.printStackTrace();
-        }
-        assertThat(assertionError + " should be null", assertionError, nullValue());
-    }
-
-    public void testConcurrentAddingAndPercolating() throws Exception {
-        assertAcked(prepareCreate("index").addMapping("type", "field1", "type=text", "field2", "type=text"));
-        ensureGreen();
-        final int numIndexThreads = scaledRandomIntBetween(1, 3);
-        final int numPercolateThreads = scaledRandomIntBetween(2, 6);
-        final int numPercolatorOperationsPerThread = scaledRandomIntBetween(100, 1000);
-
-        final Set<Throwable> exceptionsHolder = ConcurrentCollections.newConcurrentSet();
-        final CountDownLatch start = new CountDownLatch(1);
-        final AtomicInteger runningPercolateThreads = new AtomicInteger(numPercolateThreads);
-        final AtomicInteger type1 = new AtomicInteger();
-        final AtomicInteger type2 = new AtomicInteger();
-        final AtomicInteger type3 = new AtomicInteger();
-
-        final AtomicInteger idGen = new AtomicInteger();
-
-        Thread[] indexThreads = new Thread[numIndexThreads];
-        for (int i = 0; i < numIndexThreads; i++) {
-            final Random rand = new Random(getRandom().nextLong());
-            Runnable r = new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        XContentBuilder onlyField1 = XContentFactory.jsonBuilder().startObject()
-                                .field("query", termQuery("field1", "value")).endObject();
-                        XContentBuilder onlyField2 = XContentFactory.jsonBuilder().startObject()
-                                .field("query", termQuery("field2", "value")).endObject();
-                        XContentBuilder field1And2 = XContentFactory.jsonBuilder().startObject()
-                                .field("query", boolQuery().must(termQuery("field1", "value")).must(termQuery("field2", "value"))).endObject();
-
-                        start.await();
-                        while (runningPercolateThreads.get() > 0) {
-                            Thread.sleep(100);
-                            int x = rand.nextInt(3);
-                            String id = Integer.toString(idGen.incrementAndGet());
-                            IndexResponse response;
-                            switch (x) {
-                                case 0:
-                                    response = client().prepareIndex("index", PercolatorService.TYPE_NAME, id)
-                                            .setSource(onlyField1)
-                                            .setRefresh(true)
-                                            .execute().actionGet();
-                                    type1.incrementAndGet();
-                                    break;
-                                case 1:
-                                    response = client().prepareIndex("index", PercolatorService.TYPE_NAME, id)
-                                            .setSource(onlyField2)
-                                            .setRefresh(true)
-                                            .execute().actionGet();
-                                    type2.incrementAndGet();
-                                    break;
-                                case 2:
-                                    response = client().prepareIndex("index", PercolatorService.TYPE_NAME, id)
-                                            .setSource(field1And2)
-                                            .setRefresh(true)
-                                            .execute().actionGet();
-                                    type3.incrementAndGet();
-                                    break;
-                                default:
-                                    throw new IllegalStateException("Illegal x=" + x);
-                            }
-                            assertThat(response.getId(), equalTo(id));
-                            assertThat(response.getVersion(), equalTo(1L));
-                        }
-                    } catch (Throwable t) {
-                        exceptionsHolder.add(t);
-                        logger.error("Error in indexing thread...", t);
-                    }
-                }
-            };
-            indexThreads[i] = new Thread(r);
-            indexThreads[i].start();
-        }
-
-        Thread[] percolateThreads = new Thread[numPercolateThreads];
-        for (int i = 0; i < numPercolateThreads; i++) {
-            final Random rand = new Random(getRandom().nextLong());
-            Runnable r = new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        XContentBuilder onlyField1Doc = XContentFactory.jsonBuilder().startObject().startObject("doc")
-                                .field("field1", "value")
-                                .endObject().endObject();
-                        XContentBuilder onlyField2Doc = XContentFactory.jsonBuilder().startObject().startObject("doc")
-                                .field("field2", "value")
-                                .endObject().endObject();
-                        XContentBuilder field1AndField2Doc = XContentFactory.jsonBuilder().startObject().startObject("doc")
-                                .field("field1", "value")
-                                .field("field2", "value")
-                                .endObject().endObject();
-                        start.await();
-                        for (int counter = 0; counter < numPercolatorOperationsPerThread; counter++) {
-                            int x = rand.nextInt(3);
-                            int atLeastExpected;
-                            PercolateResponse response;
-                            switch (x) {
-                                case 0:
-                                    atLeastExpected = type1.get();
-                                    response = client().preparePercolate().setIndices("index").setDocumentType("type")
-                                            .setSource(onlyField1Doc).execute().actionGet();
-                                    assertNoFailures(response);
-                                    assertThat(response.getSuccessfulShards(), equalTo(response.getTotalShards()));
-                                    assertThat(response.getCount(), greaterThanOrEqualTo((long) atLeastExpected));
-                                    break;
-                                case 1:
-                                    atLeastExpected = type2.get();
-                                    response = client().preparePercolate().setIndices("index").setDocumentType("type")
-                                            .setSource(onlyField2Doc).execute().actionGet();
-                                    assertNoFailures(response);
-                                    assertThat(response.getSuccessfulShards(), equalTo(response.getTotalShards()));
-                                    assertThat(response.getCount(), greaterThanOrEqualTo((long) atLeastExpected));
-                                    break;
-                                case 2:
-                                    atLeastExpected = type3.get();
-                                    response = client().preparePercolate().setIndices("index").setDocumentType("type")
-                                            .setSource(field1AndField2Doc).execute().actionGet();
-                                    assertNoFailures(response);
-                                    assertThat(response.getSuccessfulShards(), equalTo(response.getTotalShards()));
-                                    assertThat(response.getCount(), greaterThanOrEqualTo((long) atLeastExpected));
-                                    break;
-                            }
-                        }
-                    } catch (Throwable t) {
-                        exceptionsHolder.add(t);
-                        logger.error("Error in percolate thread...", t);
-                    } finally {
-                        runningPercolateThreads.decrementAndGet();
-                    }
-                }
-            };
-            percolateThreads[i] = new Thread(r);
-            percolateThreads[i].start();
-        }
-
-        start.countDown();
-        for (Thread thread : indexThreads) {
-            thread.join();
-        }
-        for (Thread thread : percolateThreads) {
-            thread.join();
-        }
-
-        for (Throwable t : exceptionsHolder) {
-            logger.error("Unexpected exception {}", t.getMessage(), t);
-        }
-        assertThat(exceptionsHolder.isEmpty(), equalTo(true));
-    }
-
-    public void testConcurrentAddingAndRemovingWhilePercolating() throws Exception {
-        assertAcked(prepareCreate("index").addMapping("type", "field1", "type=text"));
-        ensureGreen();
-        final int numIndexThreads = scaledRandomIntBetween(1, 3);
-        final int numberPercolateOperation = scaledRandomIntBetween(10, 100);
-
-        final AtomicReference<Throwable> exceptionHolder = new AtomicReference<>(null);
-        final AtomicInteger idGen = new AtomicInteger(0);
-        final Set<String> liveIds = ConcurrentCollections.newConcurrentSet();
-        final AtomicBoolean run = new AtomicBoolean(true);
-        Thread[] indexThreads = new Thread[numIndexThreads];
-        final Semaphore semaphore = new Semaphore(numIndexThreads, true);
-        for (int i = 0; i < indexThreads.length; i++) {
-            final Random rand = new Random(getRandom().nextLong());
-            Runnable r = new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        XContentBuilder doc = XContentFactory.jsonBuilder().startObject()
-                                .field("query", termQuery("field1", "value")).endObject();
-                        outer:
-                        while (run.get()) {
-                            semaphore.acquire();
-                            try {
-                                if (!liveIds.isEmpty() && rand.nextInt(100) < 19) {
-                                    String id;
-                                    do {
-                                        if (liveIds.isEmpty()) {
-                                            continue outer;
-                                        }
-                                        id = Integer.toString(randomInt(idGen.get()));
-                                    } while (!liveIds.remove(id));
-
-                                    DeleteResponse response = client().prepareDelete("index", PercolatorService.TYPE_NAME, id)
-                                            .setRefresh(true)
-                                            .execute().actionGet();
-                                    assertThat(response.getId(), equalTo(id));
-                                    assertThat("doc[" + id + "] should have been deleted, but isn't", response.isFound(), equalTo(true));
-                                } else {
-                                    String id = Integer.toString(idGen.getAndIncrement());
-                                    IndexResponse response = client().prepareIndex("index", PercolatorService.TYPE_NAME, id)
-                                            .setSource(doc)
-                                            .setRefresh(true)
-                                            .execute().actionGet();
-                                    liveIds.add(id);
-                                    assertThat(response.isCreated(), equalTo(true)); // We only add new docs
-                                    assertThat(response.getId(), equalTo(id));
-                                }
-                            } finally {
-                                semaphore.release();
-                            }
-                        }
-                    } catch (InterruptedException iex) {
-                        logger.error("indexing thread was interrupted...", iex);
-                        run.set(false);
-                    } catch (Throwable t) {
-                        run.set(false);
-                        exceptionHolder.set(t);
-                        logger.error("Error in indexing thread...", t);
-                    }
-                }
-            };
-            indexThreads[i] = new Thread(r);
-            indexThreads[i].start();
-        }
-
-        String percolateDoc = XContentFactory.jsonBuilder().startObject()
-                .field("field1", "value")
-                .endObject().string();
-        for (int counter = 0; counter < numberPercolateOperation; counter++) {
-            Thread.sleep(5);
-            semaphore.acquire(numIndexThreads);
-            try {
-                if (!run.get()) {
-                    break;
-                }
-                int atLeastExpected = liveIds.size();
-                PercolateResponse response = client().preparePercolate().setIndices("index").setDocumentType("type")
-                        .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(percolateDoc))
-                        .setSize(atLeastExpected)
-                        .get();
-                assertThat(response.getShardFailures(), emptyArray());
-                assertThat(response.getSuccessfulShards(), equalTo(response.getTotalShards()));
-                assertThat(response.getMatches().length, equalTo(atLeastExpected));
-            } finally {
-                semaphore.release(numIndexThreads);
-            }
-        }
-        run.set(false);
-        for (Thread thread : indexThreads) {
-            thread.join();
-        }
-        assertThat("exceptionHolder should have been empty, but holds: " + exceptionHolder.toString(), exceptionHolder.get(), nullValue());
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java b/core/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
index 6dfcc5a878a..f51180f5a5a 100644
--- a/core/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
+++ b/core/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
@@ -24,8 +24,10 @@ import org.elasticsearch.action.percolate.MultiPercolateRequestBuilder;
 import org.elasticsearch.action.percolate.MultiPercolateResponse;
 import org.elasticsearch.action.percolate.PercolateSourceBuilder;
 import org.elasticsearch.client.Requests;
+import org.elasticsearch.common.compress.NotXContentException;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.percolator.PercolatorFieldMapper;
 import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -49,6 +51,7 @@ import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
 import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -60,19 +63,19 @@ public class MultiPercolatorIT extends ESIntegTestCase {
         ensureGreen();
 
         logger.info("--> register a queries");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "2")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "2")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "3")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "3")
                 .setSource(jsonBuilder().startObject().field("query", boolQuery()
                         .must(matchQuery("field1", "b"))
                         .must(matchQuery("field1", "c"))
                 ).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "4")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "4")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
                 .execute().actionGet();
         refresh();
@@ -122,7 +125,7 @@ public class MultiPercolatorIT extends ESIntegTestCase {
         item = response.getItems()[4];
         assertThat(item.getResponse(), nullValue());
         assertThat(item.getErrorMessage(), notNullValue());
-        assertThat(item.getErrorMessage(), containsString("document missing"));
+        assertThat(item.getErrorMessage(), containsString("[test/type/5] doesn't exist"));
     }
 
     public void testWithRouting() throws Exception {
@@ -130,22 +133,22 @@ public class MultiPercolatorIT extends ESIntegTestCase {
         ensureGreen();
 
         logger.info("--> register a queries");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setRouting("a")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "2")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "2")
                 .setRouting("a")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "3")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "3")
                 .setRouting("a")
                 .setSource(jsonBuilder().startObject().field("query", boolQuery()
                                 .must(matchQuery("field1", "b"))
                                 .must(matchQuery("field1", "c"))
                 ).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "4")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "4")
                 .setRouting("a")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
                 .execute().actionGet();
@@ -201,7 +204,7 @@ public class MultiPercolatorIT extends ESIntegTestCase {
         item = response.getItems()[4];
         assertThat(item.getResponse(), nullValue());
         assertThat(item.getErrorMessage(), notNullValue());
-        assertThat(item.getErrorMessage(), containsString("document missing"));
+        assertThat(item.getErrorMessage(), containsString("[test/type/5] doesn't exist"));
     }
 
     public void testExistingDocsOnly() throws Exception {
@@ -210,7 +213,7 @@ public class MultiPercolatorIT extends ESIntegTestCase {
         int numQueries = randomIntBetween(50, 100);
         logger.info("--> register a queries");
         for (int i = 0; i < numQueries; i++) {
-            client().prepareIndex("test", PercolatorService.TYPE_NAME, Integer.toString(i))
+            client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, Integer.toString(i))
                     .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
                     .execute().actionGet();
         }
@@ -253,7 +256,7 @@ public class MultiPercolatorIT extends ESIntegTestCase {
         assertThat(response.items().length, equalTo(numPercolateRequest));
         for (MultiPercolateResponse.Item item : response) {
             assertThat(item.isFailure(), equalTo(true));
-            assertThat(item.getErrorMessage(), containsString("document missing"));
+            assertThat(item.getErrorMessage(), containsString("doesn't exist"));
             assertThat(item.getResponse(), nullValue());
         }
 
@@ -283,12 +286,10 @@ public class MultiPercolatorIT extends ESIntegTestCase {
         createIndex("test");
         ensureGreen();
 
-        NumShards test = getNumShards("test");
-
         int numQueries = randomIntBetween(50, 100);
         logger.info("--> register a queries");
         for (int i = 0; i < numQueries; i++) {
-            client().prepareIndex("test", PercolatorService.TYPE_NAME, Integer.toString(i))
+            client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, Integer.toString(i))
                     .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
                     .execute().actionGet();
         }
@@ -324,13 +325,8 @@ public class MultiPercolatorIT extends ESIntegTestCase {
         response = builder.execute().actionGet();
         assertThat(response.items().length, equalTo(numPercolateRequest));
         for (MultiPercolateResponse.Item item : response) {
-            assertThat(item.isFailure(), equalTo(false));
-            assertThat(item.getResponse().getSuccessfulShards(), equalTo(0));
-            assertThat(item.getResponse().getShardFailures().length, equalTo(test.numPrimaries));
-            for (ShardOperationFailedException shardFailure : item.getResponse().getShardFailures()) {
-                assertThat(shardFailure.reason(), containsString("Failed to derive xcontent"));
-                assertThat(shardFailure.status().getStatus(), equalTo(400));
-            }
+            assertThat(item.isFailure(), equalTo(true));
+            assertThat(item.getFailure(), notNullValue());
         }
 
         // one valid request
@@ -402,7 +398,7 @@ public class MultiPercolatorIT extends ESIntegTestCase {
         assertAcked(client().admin().indices().prepareCreate("nestedindex").addMapping("company", mapping));
         ensureGreen("nestedindex");
 
-        client().prepareIndex("nestedindex", PercolatorService.TYPE_NAME, "Q").setSource(jsonBuilder().startObject()
+        client().prepareIndex("nestedindex", PercolatorFieldMapper.TYPE_NAME, "Q").setSource(jsonBuilder().startObject()
                 .field("query", QueryBuilders.nestedQuery("employee", QueryBuilders.matchQuery("employee.name", "virginia potts").operator(Operator.AND)).scoreMode(ScoreMode.Avg)).endObject()).get();
 
         refresh();
diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolateDocumentParserTests.java b/core/src/test/java/org/elasticsearch/percolator/PercolateDocumentParserTests.java
deleted file mode 100644
index 854a25358e7..00000000000
--- a/core/src/test/java/org/elasticsearch/percolator/PercolateDocumentParserTests.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.percolator;
-
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.TermQuery;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.percolate.PercolateShardRequest;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisService;
-import org.elasticsearch.index.analysis.AnalyzerProvider;
-import org.elasticsearch.index.analysis.CharFilterFactory;
-import org.elasticsearch.index.analysis.TokenFilterFactory;
-import org.elasticsearch.index.analysis.TokenizerFactory;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.query.QueryParser;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.query.TermQueryParser;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.similarity.SimilarityService;
-import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.aggregations.AggregationBinaryParseElement;
-import org.elasticsearch.search.aggregations.AggregationParseElement;
-import org.elasticsearch.search.aggregations.AggregationPhase;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
-import org.elasticsearch.search.highlight.HighlightPhase;
-import org.elasticsearch.search.highlight.Highlighters;
-import org.elasticsearch.search.sort.SortParseElement;
-import org.elasticsearch.test.ESTestCase;
-import org.junit.Before;
-import org.mockito.Mockito;
-
-import java.util.Collections;
-import java.util.Map;
-
-import static java.util.Collections.singletonMap;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.nullValue;
-
-public class PercolateDocumentParserTests extends ESTestCase {
-
-    private MapperService mapperService;
-    private PercolateDocumentParser parser;
-    private QueryShardContext queryShardContext;
-    private PercolateShardRequest request;
-
-    @Before
-    public void init() {
-        IndexSettings indexSettings = new IndexSettings(new IndexMetaData.Builder("_index").settings(
-                Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
-                        .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
-                        .build(), Settings.EMPTY);
-        AnalysisService analysisService = new AnalysisService(indexSettings, Collections.<String, AnalyzerProvider>emptyMap(), Collections.<String, TokenizerFactory>emptyMap(), Collections.<String, CharFilterFactory>emptyMap(), Collections.<String, TokenFilterFactory>emptyMap());
-        IndicesModule indicesModule = new IndicesModule();
-        mapperService = new MapperService(indexSettings, analysisService, new SimilarityService(indexSettings, Collections.emptyMap()), indicesModule.getMapperRegistry(), () -> null);
-
-        Map<String, QueryParser<?>> parsers = singletonMap("term", new TermQueryParser());
-        IndicesQueriesRegistry indicesQueriesRegistry = new IndicesQueriesRegistry(indexSettings.getSettings(), parsers);
-
-        queryShardContext = new QueryShardContext(indexSettings, null, null, mapperService, null, null, indicesQueriesRegistry);
-
-        HighlightPhase highlightPhase = new HighlightPhase(Settings.EMPTY, new Highlighters());
-        AggregatorParsers aggregatorParsers = new AggregatorParsers(Collections.emptySet(), Collections.emptySet(),
-                new NamedWriteableRegistry());
-        AggregationPhase aggregationPhase = new AggregationPhase(new AggregationParseElement(aggregatorParsers, indicesQueriesRegistry),
-                new AggregationBinaryParseElement(aggregatorParsers, indicesQueriesRegistry));
-        parser = new PercolateDocumentParser(highlightPhase, new SortParseElement(), aggregationPhase);
-
-        request = Mockito.mock(PercolateShardRequest.class);
-        Mockito.when(request.shardId()).thenReturn(new ShardId("_index", "_na_", 0));
-        Mockito.when(request.documentType()).thenReturn("type");
-    }
-
-    public void testParseDoc() throws Exception {
-        XContentBuilder source = jsonBuilder().startObject()
-                .startObject("doc")
-                    .field("field1", "value1")
-                .endObject()
-                .endObject();
-        Mockito.when(request.source()).thenReturn(source.bytes());
-
-        PercolateContext context = new PercolateContext(request, new SearchShardTarget("_node", new Index("_index", "_na_"), 0), mapperService, queryShardContext);
-        ParsedDocument parsedDocument = parser.parse(request, context, mapperService);
-        assertThat(parsedDocument.rootDoc().get("field1"), equalTo("value1"));
-    }
-
-    public void testParseDocAndOtherOptions() throws Exception {
-        XContentBuilder source = jsonBuilder().startObject()
-                .startObject("doc")
-                    .field("field1", "value1")
-                .endObject()
-                .startObject("query")
-                    .startObject("term").field("field1", "value1").endObject()
-                .endObject()
-                .field("track_scores", true)
-                .field("size", 123)
-                .startObject("sort").startObject("_score").endObject().endObject()
-                .endObject();
-        Mockito.when(request.source()).thenReturn(source.bytes());
-
-        PercolateContext context = new PercolateContext(request, new SearchShardTarget("_node", new Index("_index", "_na_"), 0), mapperService, queryShardContext);
-        ParsedDocument parsedDocument = parser.parse(request, context, mapperService);
-        assertThat(parsedDocument.rootDoc().get("field1"), equalTo("value1"));
-        assertThat(context.percolateQuery(), equalTo(new TermQuery(new Term("field1", "value1"))));
-        assertThat(context.trackScores(), is(true));
-        assertThat(context.size(), is(123));
-        assertThat(context.sort(), nullValue());
-    }
-
-    public void testParseDocSource() throws Exception {
-        XContentBuilder source = jsonBuilder().startObject()
-                .startObject("query")
-                .startObject("term").field("field1", "value1").endObject()
-                .endObject()
-                .field("track_scores", true)
-                .field("size", 123)
-                .startObject("sort").startObject("_score").endObject().endObject()
-                .endObject();
-        XContentBuilder docSource = jsonBuilder().startObject()
-                .field("field1", "value1")
-                .endObject();
-        Mockito.when(request.source()).thenReturn(source.bytes());
-        Mockito.when(request.docSource()).thenReturn(docSource.bytes());
-
-        PercolateContext context = new PercolateContext(request, new SearchShardTarget("_node", new Index("_index", "_na_"), 0), mapperService, queryShardContext);
-        ParsedDocument parsedDocument = parser.parse(request, context, mapperService);
-        assertThat(parsedDocument.rootDoc().get("field1"), equalTo("value1"));
-        assertThat(context.percolateQuery(), equalTo(new TermQuery(new Term("field1", "value1"))));
-        assertThat(context.trackScores(), is(true));
-        assertThat(context.size(), is(123));
-        assertThat(context.sort(), nullValue());
-    }
-
-    public void testParseDocSourceAndSource() throws Exception {
-        XContentBuilder source = jsonBuilder().startObject()
-                .startObject("doc")
-                .field("field1", "value1")
-                .endObject()
-                .startObject("query")
-                .startObject("term").field("field1", "value1").endObject()
-                .endObject()
-                .field("track_scores", true)
-                .field("size", 123)
-                .startObject("sort").startObject("_score").endObject().endObject()
-                .endObject();
-        XContentBuilder docSource = jsonBuilder().startObject()
-                .field("field1", "value1")
-                .endObject();
-        Mockito.when(request.source()).thenReturn(source.bytes());
-        Mockito.when(request.docSource()).thenReturn(docSource.bytes());
-
-        PercolateContext context = new PercolateContext(request, new SearchShardTarget("_node", new Index("_index", "_na_"), 0), mapperService, queryShardContext);
-        try {
-            parser.parse(request, context, mapperService);
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("Can't specify the document to percolate in the source of the request and as document id"));
-        }
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java b/core/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java
index f07e50b1dfc..8393c80786c 100644
--- a/core/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java
+++ b/core/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java
@@ -21,6 +21,7 @@ package org.elasticsearch.percolator;
 import org.elasticsearch.action.percolate.PercolateRequestBuilder;
 import org.elasticsearch.action.percolate.PercolateResponse;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.percolator.PercolatorFieldMapper;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
@@ -69,7 +70,7 @@ public class PercolatorAggregationsIT extends ESIntegTestCase {
             String value = values[i % numUniqueQueries];
             expectedCount[i % numUniqueQueries]++;
             QueryBuilder queryBuilder = matchQuery("field1", value);
-            client().prepareIndex("test", PercolatorService.TYPE_NAME, Integer.toString(i))
+            client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, Integer.toString(i))
                     .setSource(jsonBuilder().startObject().field("query", queryBuilder).field("field2", "b").endObject()).execute()
                     .actionGet();
         }
@@ -134,7 +135,7 @@ public class PercolatorAggregationsIT extends ESIntegTestCase {
             String value = values[i % numUniqueQueries];
             expectedCount[i % numUniqueQueries]++;
             QueryBuilder queryBuilder = matchQuery("field1", value);
-            client().prepareIndex("test", PercolatorService.TYPE_NAME, Integer.toString(i))
+            client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, Integer.toString(i))
                     .setSource(jsonBuilder().startObject().field("query", queryBuilder).field("field2", "b").endObject()).execute()
                     .actionGet();
         }
@@ -212,7 +213,7 @@ public class PercolatorAggregationsIT extends ESIntegTestCase {
         for (int i = 0; i < numQueries; i++) {
             String value = "value0";
             QueryBuilder queryBuilder = matchQuery("field1", value);
-            client().prepareIndex("test", PercolatorService.TYPE_NAME, Integer.toString(i))
+            client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, Integer.toString(i))
                     .setSource(jsonBuilder().startObject().field("query", queryBuilder).field("field2", i % 3 == 0 ? "b" : "a").endObject())
                     .execute()
                     .actionGet();
diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityIT.java
index 9378eaa7542..a52dfa48c32 100644
--- a/core/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityIT.java
+++ b/core/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityIT.java
@@ -68,7 +68,7 @@ public class PercolatorBackwardsCompatibilityIT extends ESIntegTestCase {
         // verify existing percolator queries:
         SearchResponse searchResponse = client().prepareSearch(INDEX_NAME)
             .setTypes(".percolator")
-            .addSort("_id", SortOrder.ASC)
+            .addSort("_uid", SortOrder.ASC)
             .get();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L));
         assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1"));
diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java b/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
index 01494aab72d..f6f5d260550 100644
--- a/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
+++ b/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
@@ -20,11 +20,9 @@ package org.elasticsearch.percolator;
 
 import com.vividsolutions.jts.geom.Coordinate;
 import org.apache.lucene.search.join.ScoreMode;
-import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
-import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.percolate.PercolateResponse;
 import org.elasticsearch.action.percolate.PercolateSourceBuilder;
@@ -38,17 +36,15 @@ import org.elasticsearch.common.settings.Settings.Builder;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.engine.DocumentMissingException;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.percolator.PercolatorFieldMapper;
 import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.functionscore.weight.WeightBuilder;
 import org.elasticsearch.index.query.support.QueryInnerHits;
-import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.highlight.HighlightBuilder;
-import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.io.IOException;
@@ -83,7 +79,6 @@ import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction;
 import static org.elasticsearch.percolator.PercolatorTestUtil.convertFromTextArray;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertMatchCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
@@ -94,10 +89,7 @@ import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.emptyArray;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 /**
@@ -113,19 +105,19 @@ public class PercolatorIT extends ESIntegTestCase {
         client().prepareIndex("test", "type", "1").setSource("field1", "value").execute().actionGet();
 
         logger.info("--> register a queries");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "2")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "2")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "3")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "3")
                 .setSource(jsonBuilder().startObject().field("query", boolQuery()
                         .must(matchQuery("field1", "b"))
                         .must(matchQuery("field1", "c"))
                 ).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "4")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "4")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
                 .execute().actionGet();
         refresh();
@@ -173,7 +165,8 @@ public class PercolatorIT extends ESIntegTestCase {
                     .setGetRequest(Requests.getRequest("test").type("type").id("5"))
                     .execute().actionGet();
             fail("Exception should have been thrown");
-        } catch (DocumentMissingException e) {
+        } catch (ResourceNotFoundException e) {
+            assertThat(e.getMessage(), equalTo("percolate document [test/type/5] doesn't exist"));
         }
     }
 
@@ -194,7 +187,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getMatches(), emptyArray());
 
         // add first query...
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "test1")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "test1")
                 .setSource(XContentFactory.jsonBuilder().startObject().field("query", termQuery("field2", "value")).endObject())
                 .execute().actionGet();
         refresh();
@@ -207,7 +200,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(convertFromTextArray(response.getMatches(), "test"), arrayContaining("test1"));
 
         // add second query...
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "test2")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "test2")
                 .setSource(XContentFactory.jsonBuilder().startObject().field("query", termQuery("field1", 1)).endObject())
                 .execute().actionGet();
         refresh();
@@ -221,7 +214,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(convertFromTextArray(response.getMatches(), "test"), arrayContainingInAnyOrder("test1", "test2"));
 
 
-        client().prepareDelete("test", PercolatorService.TYPE_NAME, "test2").execute().actionGet();
+        client().prepareDelete("test", PercolatorFieldMapper.TYPE_NAME, "test2").execute().actionGet();
         refresh();
         response = client().preparePercolate()
                 .setIndices("test").setDocumentType("type1")
@@ -239,7 +232,7 @@ public class PercolatorIT extends ESIntegTestCase {
 
         logger.info("--> register a queries");
         for (int i = 1; i <= 100; i++) {
-            client().prepareIndex("test", PercolatorService.TYPE_NAME, Integer.toString(i))
+            client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, Integer.toString(i))
                     .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
                     .setRouting(Integer.toString(i % 2))
                     .execute().actionGet();
@@ -282,7 +275,7 @@ public class PercolatorIT extends ESIntegTestCase {
 
         client().prepareIndex("my-queries-index", "test", "1").setSource("field1", "value1").execute().actionGet();
         logger.info("--> register a query");
-        client().prepareIndex("my-queries-index", PercolatorService.TYPE_NAME, "kuku1")
+        client().prepareIndex("my-queries-index", PercolatorFieldMapper.TYPE_NAME, "kuku1")
                 .setSource(jsonBuilder().startObject()
                         .field("color", "blue")
                         .field("query", termQuery("field1", "value1"))
@@ -296,7 +289,7 @@ public class PercolatorIT extends ESIntegTestCase {
 
         client().prepareIndex("my-queries-index", "test", "1").setSource("field1", "value1").execute().actionGet();
         logger.info("--> register a query");
-        client().prepareIndex("my-queries-index", PercolatorService.TYPE_NAME, "kuku2")
+        client().prepareIndex("my-queries-index", PercolatorFieldMapper.TYPE_NAME, "kuku2")
                 .setSource(jsonBuilder().startObject()
                         .field("color", "blue")
                         .field("query", termQuery("field1", "value1"))
@@ -323,7 +316,7 @@ public class PercolatorIT extends ESIntegTestCase {
         ensureGreen();
 
         logger.info("--> register a query");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject()
                         .field("source", "productizer")
                         .field("query", QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("filingcategory:s")))
@@ -351,7 +344,7 @@ public class PercolatorIT extends ESIntegTestCase {
         ensureGreen();
 
         logger.info("--> register a query");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "kuku")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "kuku")
                 .setSource(jsonBuilder().startObject()
                         .field("color", "blue")
                         .field("query", termQuery("field1", "value1"))
@@ -359,7 +352,7 @@ public class PercolatorIT extends ESIntegTestCase {
                 .execute().actionGet();
         refresh();
         SearchResponse countResponse = client().prepareSearch().setSize(0)
-                .setQuery(matchAllQuery()).setTypes(PercolatorService.TYPE_NAME)
+                .setQuery(matchAllQuery()).setTypes(PercolatorFieldMapper.TYPE_NAME)
                 .execute().actionGet();
         assertThat(countResponse.getHits().totalHits(), equalTo(1L));
 
@@ -388,7 +381,7 @@ public class PercolatorIT extends ESIntegTestCase {
         client().admin().indices().prepareDelete("test").execute().actionGet();
         logger.info("--> make sure percolated queries for it have been deleted as well");
         countResponse = client().prepareSearch().setSize(0)
-                .setQuery(matchAllQuery()).setTypes(PercolatorService.TYPE_NAME)
+                .setQuery(matchAllQuery()).setTypes(PercolatorFieldMapper.TYPE_NAME)
                 .execute().actionGet();
         assertHitCount(countResponse, 0L);
     }
@@ -398,7 +391,7 @@ public class PercolatorIT extends ESIntegTestCase {
         ensureGreen();
 
         logger.info("--> register a query 1");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "kuku")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "kuku")
                 .setSource(jsonBuilder().startObject()
                         .field("color", "blue")
                         .field("query", termQuery("field1", "value1"))
@@ -407,7 +400,7 @@ public class PercolatorIT extends ESIntegTestCase {
                 .execute().actionGet();
 
         logger.info("--> register a query 2");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "bubu")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "bubu")
                 .setSource(jsonBuilder().startObject()
                         .field("color", "green")
                         .field("query", termQuery("field1", "value2"))
@@ -441,7 +434,7 @@ public class PercolatorIT extends ESIntegTestCase {
         ensureGreen();
 
         logger.info("--> register a query 1");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "kuku")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "kuku")
                 .setSource(jsonBuilder().startObject()
                         .field("color", "blue")
                         .field("query", termQuery("field1", "value1"))
@@ -458,7 +451,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(convertFromTextArray(percolate.getMatches(), "test"), arrayContaining("kuku"));
 
         logger.info("--> register a query 2");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "bubu")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "bubu")
                 .setSource(jsonBuilder().startObject()
                         .field("color", "green")
                         .field("query", termQuery("field1", "value2"))
@@ -475,7 +468,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(convertFromTextArray(percolate.getMatches(), "test"), arrayContaining("bubu"));
 
         logger.info("--> register a query 3");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "susu")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "susu")
                 .setSource(jsonBuilder().startObject()
                         .field("color", "red")
                         .field("query", termQuery("field1", "value2"))
@@ -495,7 +488,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(convertFromTextArray(percolate.getMatches(), "test"), arrayContaining("susu"));
 
         logger.info("--> deleting query 1");
-        client().prepareDelete("test", PercolatorService.TYPE_NAME, "kuku").setRefresh(true).execute().actionGet();
+        client().prepareDelete("test", PercolatorFieldMapper.TYPE_NAME, "kuku").setRefresh(true).execute().actionGet();
 
         percolate = client().preparePercolate()
                 .setIndices("test").setDocumentType("type1")
@@ -512,7 +505,10 @@ public class PercolatorIT extends ESIntegTestCase {
         ensureGreen();
 
         logger.info("--> register a query");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
+                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
+                .execute().actionGet();
+        client().prepareIndex("test2", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
                 .execute().actionGet();
         refresh();
@@ -528,67 +524,7 @@ public class PercolatorIT extends ESIntegTestCase {
         NumShards numShards = getNumShards("test");
 
         IndicesStatsResponse indicesResponse = client().admin().indices().prepareStats("test").execute().actionGet();
-        assertThat(indicesResponse.getTotal().getPercolate().getCount(), equalTo((long) numShards.numPrimaries));
-        assertThat(indicesResponse.getTotal().getPercolate().getCurrent(), equalTo(0L));
-        assertThat(indicesResponse.getTotal().getPercolate().getNumQueries(), equalTo((long)numShards.dataCopies)); //number of copies
-        assertThat(indicesResponse.getTotal().getPercolate().getMemorySizeInBytes(), equalTo(-1L));
-
-        NodesStatsResponse nodesResponse = client().admin().cluster().prepareNodesStats().execute().actionGet();
-        long percolateCount = 0;
-        for (NodeStats nodeStats : nodesResponse) {
-            percolateCount += nodeStats.getIndices().getPercolate().getCount();
-        }
-        assertThat(percolateCount, equalTo((long) numShards.numPrimaries));
-
-        logger.info("--> Second percolate request");
-        response = client().preparePercolate()
-                .setIndices("test").setDocumentType("type")
-                .setSource(jsonBuilder().startObject().startObject("doc").field("field", "val").endObject().endObject())
-                .execute().actionGet();
-        assertMatchCount(response, 1L);
-        assertThat(response.getMatches(), arrayWithSize(1));
-        assertThat(convertFromTextArray(response.getMatches(), "test"), arrayContaining("1"));
-
-        indicesResponse = client().admin().indices().prepareStats().setPercolate(true).execute().actionGet();
-        assertThat(indicesResponse.getTotal().getPercolate().getCount(), equalTo((long) numShards.numPrimaries * 2));
-        assertThat(indicesResponse.getTotal().getPercolate().getCurrent(), equalTo(0L));
-        assertThat(indicesResponse.getTotal().getPercolate().getNumQueries(), equalTo((long)numShards.dataCopies)); //number of copies
-        assertThat(indicesResponse.getTotal().getPercolate().getMemorySizeInBytes(), equalTo(-1L));
-
-        percolateCount = 0;
-        nodesResponse = client().admin().cluster().prepareNodesStats().execute().actionGet();
-        for (NodeStats nodeStats : nodesResponse) {
-            percolateCount += nodeStats.getIndices().getPercolate().getCount();
-        }
-        assertThat(percolateCount, equalTo((long) numShards.numPrimaries *2));
-
-        // We might be faster than 1 ms, so run upto 1000 times until have spend 1ms or more on percolating
-        boolean moreThanOneMs = false;
-        int counter = 3; // We already ran two times.
-        do {
-            indicesResponse = client().admin().indices().prepareStats("test").execute().actionGet();
-            if (indicesResponse.getTotal().getPercolate().getTimeInMillis() > 0) {
-                moreThanOneMs = true;
-                break;
-            }
-
-            logger.info("--> {}th percolate request", counter);
-            response = client().preparePercolate()
-                    .setIndices("test").setDocumentType("type")
-                    .setSource(jsonBuilder().startObject().startObject("doc").field("field", "val").endObject().endObject())
-                    .execute().actionGet();
-            assertThat(response.getMatches(), arrayWithSize(1));
-            assertThat(convertFromTextArray(response.getMatches(), "test"), arrayContaining("1"));
-        } while (++counter <= 1000);
-        assertTrue("Something is off, we should have spent at least 1ms on percolating...", moreThanOneMs);
-
-        long percolateSumTime = 0;
-        nodesResponse = client().admin().cluster().prepareNodesStats().execute().actionGet();
-        for (NodeStats nodeStats : nodesResponse) {
-            percolateCount += nodeStats.getIndices().getPercolate().getCount();
-            percolateSumTime += nodeStats.getIndices().getPercolate().getTimeInMillis();
-        }
-        assertThat(percolateSumTime, greaterThan(0L));
+        assertThat(indicesResponse.getTotal().getPercolatorCache().getNumQueries(), equalTo((long)numShards.dataCopies)); // number of copies
     }
 
     public void testPercolatingExistingDocs() throws Exception {
@@ -602,19 +538,19 @@ public class PercolatorIT extends ESIntegTestCase {
         client().prepareIndex("test", "type", "4").setSource("field1", "d").execute().actionGet();
 
         logger.info("--> register a queries");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "2")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "2")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "3")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "3")
                 .setSource(jsonBuilder().startObject().field("query", boolQuery()
                         .must(matchQuery("field1", "b"))
                         .must(matchQuery("field1", "c"))
                 ).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "4")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "4")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
                 .execute().actionGet();
         refresh();
@@ -667,19 +603,19 @@ public class PercolatorIT extends ESIntegTestCase {
         client().prepareIndex("test", "type", "4").setSource("field1", "d").setRouting("1").execute().actionGet();
 
         logger.info("--> register a queries");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "2")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "2")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "3")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "3")
                 .setSource(jsonBuilder().startObject().field("query", boolQuery()
                         .must(matchQuery("field1", "b"))
                         .must(matchQuery("field1", "c"))
                 ).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "4")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "4")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
                 .execute().actionGet();
         refresh();
@@ -732,19 +668,19 @@ public class PercolatorIT extends ESIntegTestCase {
         client().prepareIndex("test", "type", "4").setSource("field1", "d").execute().actionGet();
 
         logger.info("--> registering queries");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "2")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "2")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "3")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "3")
                 .setSource(jsonBuilder().startObject().field("query", boolQuery()
                         .must(matchQuery("field1", "b"))
                         .must(matchQuery("field1", "c"))
                 ).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "4")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "4")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
                 .execute().actionGet();
         refresh();
@@ -788,7 +724,7 @@ public class PercolatorIT extends ESIntegTestCase {
         logger.info("--> registering queries");
         for (int i = 1; i <= 10; i++) {
             String index = i % 2 == 0 ? "test1" : "test2";
-            client().prepareIndex(index, PercolatorService.TYPE_NAME, Integer.toString(i))
+            client().prepareIndex(index, PercolatorFieldMapper.TYPE_NAME, Integer.toString(i))
                     .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
                     .execute().actionGet();
         }
@@ -861,15 +797,15 @@ public class PercolatorIT extends ESIntegTestCase {
 
     public void testPercolateWithAliasFilter() throws Exception {
         assertAcked(prepareCreate("my-index")
-                        .addMapping(PercolatorService.TYPE_NAME, "a", "type=keyword")
+                        .addMapping(PercolatorFieldMapper.TYPE_NAME, "a", "type=keyword")
                         .addAlias(new Alias("a").filter(QueryBuilders.termQuery("a", "a")))
                         .addAlias(new Alias("b").filter(QueryBuilders.termQuery("a", "b")))
                         .addAlias(new Alias("c").filter(QueryBuilders.termQuery("a", "c")))
         );
-        client().prepareIndex("my-index", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("my-index", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("a", "a").endObject())
                 .get();
-        client().prepareIndex("my-index", PercolatorService.TYPE_NAME, "2")
+        client().prepareIndex("my-index", PercolatorFieldMapper.TYPE_NAME, "2")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("a", "b").endObject())
                 .get();
         refresh();
@@ -944,19 +880,19 @@ public class PercolatorIT extends ESIntegTestCase {
         client().prepareIndex("test", "type", "1").setSource("field1", "value").execute().actionGet();
 
         logger.info("--> register a queries");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "2")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "2")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "3")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "3")
                 .setSource(jsonBuilder().startObject().field("query", boolQuery()
                         .must(matchQuery("field1", "b"))
                         .must(matchQuery("field1", "c"))
                 ).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "4")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "4")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
                 .execute().actionGet();
         refresh();
@@ -1000,7 +936,8 @@ public class PercolatorIT extends ESIntegTestCase {
                     .setGetRequest(Requests.getRequest("test").type("type").id("5"))
                     .execute().actionGet();
             fail("Exception should have been thrown");
-        } catch (DocumentMissingException e) {
+        } catch (ResourceNotFoundException e) {
+            assertThat(e.getMessage(), equalTo("percolate document [test/type/5] doesn't exist"));
         }
     }
 
@@ -1015,19 +952,19 @@ public class PercolatorIT extends ESIntegTestCase {
         client().prepareIndex("test", "type", "4").setSource("field1", "d").execute().actionGet();
 
         logger.info("--> register a queries");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "2")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "2")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "3")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "3")
                 .setSource(jsonBuilder().startObject().field("query", boolQuery()
                         .must(matchQuery("field1", "b"))
                         .must(matchQuery("field1", "c"))
                 ).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "4")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "4")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
                 .execute().actionGet();
         refresh();
@@ -1072,10 +1009,10 @@ public class PercolatorIT extends ESIntegTestCase {
         int numLevels = randomIntBetween(1, 25);
         long numQueriesPerLevel = randomIntBetween(10, 250);
         long totalQueries = numLevels * numQueriesPerLevel;
-        logger.info("--> register " + totalQueries + " queries");
+        logger.info("--> register {} queries", totalQueries);
         for (int level = 1; level <= numLevels; level++) {
             for (int query = 1; query <= numQueriesPerLevel; query++) {
-                client().prepareIndex("my-index", PercolatorService.TYPE_NAME, level + "-" + query)
+                client().prepareIndex("my-index", PercolatorFieldMapper.TYPE_NAME, level + "-" + query)
                         .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("level", level).endObject())
                         .execute().actionGet();
             }
@@ -1166,10 +1103,10 @@ public class PercolatorIT extends ESIntegTestCase {
 
         Map<Integer, NavigableSet<Integer>> controlMap = new HashMap<>();
         long numQueries = randomIntBetween(100, 250);
-        logger.info("--> register " + numQueries + " queries");
+        logger.info("--> register {} queries", numQueries);
         for (int i = 0; i < numQueries; i++) {
             int value = randomInt(10);
-            client().prepareIndex("my-index", PercolatorService.TYPE_NAME, Integer.toString(i))
+            client().prepareIndex("my-index", PercolatorFieldMapper.TYPE_NAME, Integer.toString(i))
                     .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("level", i).field("field1", value).endObject())
                     .execute().actionGet();
             if (!controlMap.containsKey(value)) {
@@ -1250,10 +1187,10 @@ public class PercolatorIT extends ESIntegTestCase {
         createIndex("my-index");
         ensureGreen();
 
-        client().prepareIndex("my-index", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("my-index", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("level", 1).endObject())
                 .execute().actionGet();
-        client().prepareIndex("my-index", PercolatorService.TYPE_NAME, "2")
+        client().prepareIndex("my-index", PercolatorFieldMapper.TYPE_NAME, "2")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("level", 2).endObject())
                 .execute().actionGet();
         refresh();
@@ -1271,33 +1208,6 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getMatches()[1].getScore(), equalTo(1f));
     }
 
-    public void testPercolateSortingUnsupportedField() throws Exception {
-        client().admin().indices().prepareCreate("my-index")
-                .addMapping("my-type", "field", "type=text")
-                .addMapping(PercolatorService.TYPE_NAME, "level", "type=integer", "query", "type=object,enabled=false")
-                .get();
-        ensureGreen();
-
-        client().prepareIndex("my-index", PercolatorService.TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("level", 1).endObject())
-                .get();
-        client().prepareIndex("my-index", PercolatorService.TYPE_NAME, "2")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("level", 2).endObject())
-                .get();
-        refresh();
-
-        PercolateResponse response = client().preparePercolate().setIndices("my-index").setDocumentType("my-type")
-                .setSize(2)
-                .setPercolateDoc(docBuilder().setDoc("field", "value"))
-                .setPercolateQuery(QueryBuilders.functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("level")))
-                .addSort(SortBuilders.fieldSort("level"))
-                .get();
-
-        assertThat(response.getShardFailures().length, equalTo(getNumShards("my-index").numPrimaries));
-        assertThat(response.getShardFailures()[0].status(), equalTo(RestStatus.BAD_REQUEST));
-        assertThat(response.getShardFailures()[0].reason(), containsString("Only _score desc is supported"));
-    }
-
     public void testPercolateOnEmptyIndex() throws Exception {
         client().admin().indices().prepareCreate("my-index").execute().actionGet();
         ensureGreen();
@@ -1306,7 +1216,7 @@ public class PercolatorIT extends ESIntegTestCase {
                 .setSortByScore(true)
                 .setSize(2)
                 .setPercolateDoc(docBuilder().setDoc("field", "value"))
-                .setPercolateQuery(QueryBuilders.functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("level")))
+                .setPercolateQuery(QueryBuilders.functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("level").missing(0.0)))
                 .execute().actionGet();
         assertMatchCount(response, 0L);
     }
@@ -1322,19 +1232,19 @@ public class PercolatorIT extends ESIntegTestCase {
         assertAcked(prepareCreate("test").addMapping("type", "field1", fieldMapping.toString()));
 
         logger.info("--> register a queries");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "brown fox")).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "2")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "2")
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "lazy dog")).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "3")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "3")
                 .setSource(jsonBuilder().startObject().field("query", termQuery("field1", "jumps")).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "4")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "4")
                 .setSource(jsonBuilder().startObject().field("query", termQuery("field1", "dog")).endObject())
                 .execute().actionGet();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "5")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "5")
                 .setSource(jsonBuilder().startObject().field("query", termQuery("field1", "fox")).endObject())
                 .execute().actionGet();
         refresh();
@@ -1525,7 +1435,7 @@ public class PercolatorIT extends ESIntegTestCase {
         ensureGreen();
 
         logger.info("--> register a query");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject()
                         .field("query", QueryBuilders.constantScoreQuery(QueryBuilders.boolQuery()
                                 .must(QueryBuilders.queryStringQuery("root"))
@@ -1579,10 +1489,10 @@ public class PercolatorIT extends ESIntegTestCase {
         client().admin().indices().prepareCreate("test").execute().actionGet();
         ensureGreen();
 
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
                 .get();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
                 .get();
         refresh();
@@ -1602,7 +1512,7 @@ public class PercolatorIT extends ESIntegTestCase {
         ensureGreen();
 
         try {
-            client().prepareIndex("test", PercolatorService.TYPE_NAME)
+            client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME)
                     .setSource(jsonBuilder().startObject().field("query", termQuery("field1", "value")).endObject())
                     .get();
             fail();
@@ -1611,7 +1521,7 @@ public class PercolatorIT extends ESIntegTestCase {
         }
 
         try {
-            client().prepareIndex("test", PercolatorService.TYPE_NAME)
+            client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME)
                     .setSource(jsonBuilder().startObject().field("query", rangeQuery("field1").from(0).to(1)).endObject())
                     .get();
             fail();
@@ -1626,10 +1536,10 @@ public class PercolatorIT extends ESIntegTestCase {
                 .get();
         ensureGreen();
 
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject().field("query", rangeQuery("timestamp").from("now-1d").to("now")).endObject())
                 .get();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "2")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "2")
                 .setSource(jsonBuilder().startObject().field("query", constantScoreQuery(rangeQuery("timestamp").from("now-1d").to("now"))).endObject())
                 .get();
         refresh();
@@ -1654,7 +1564,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertAcked(client().admin().indices().prepareCreate("nestedindex").addMapping("company", mapping));
         ensureGreen("nestedindex");
 
-        client().prepareIndex("nestedindex", PercolatorService.TYPE_NAME, "Q").setSource(jsonBuilder().startObject()
+        client().prepareIndex("nestedindex", PercolatorFieldMapper.TYPE_NAME, "Q").setSource(jsonBuilder().startObject()
                 .field("query", QueryBuilders.nestedQuery("employee", QueryBuilders.matchQuery("employee.name", "virginia potts").operator(Operator.AND)).scoreMode(ScoreMode.Avg)).endObject()).get();
 
         refresh();
@@ -1783,12 +1693,12 @@ public class PercolatorIT extends ESIntegTestCase {
                 "}";
         assertAcked(client().admin().indices().prepareCreate("test").addMapping("doc", mapping));
         ensureGreen("test");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME).setSource(q1).setId("q1").get();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME).setSource(q2).setId("q2").get();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME).setSource(q3).setId("q3").get();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME).setSource(q4).setId("q4").get();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME).setSource(q5).setId("q5").get();
-        client().prepareIndex("test", PercolatorService.TYPE_NAME).setSource(q6).setId("q6").get();
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME).setSource(q1).setId("q1").get();
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME).setSource(q2).setId("q2").get();
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME).setSource(q3).setId("q3").get();
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME).setSource(q4).setId("q4").get();
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME).setSource(q5).setId("q5").get();
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME).setSource(q6).setId("q6").get();
         refresh();
         PercolateResponse response = client().preparePercolate()
                 .setIndices("test").setDocumentType("doc")
@@ -1822,7 +1732,7 @@ public class PercolatorIT extends ESIntegTestCase {
                 .put("index.percolator.map_unmapped_fields_as_string", true);
         assertAcked(prepareCreate("test")
                 .setSettings(settings));
-        client().prepareIndex("test", PercolatorService.TYPE_NAME)
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME)
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "value")).endObject()).get();
         refresh();
         logger.info("--> Percolate doc with field1=value");
@@ -1842,7 +1752,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertAcked(prepareCreate("test")
             .setSettings(settings)
             .addMapping("type", "location", "type=geo_shape"));
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
             .setSource(jsonBuilder().startObject().field("query", geoShapeQuery("location", ShapeBuilders.newEnvelope(new Coordinate(0d, 50d), new Coordinate(2d, 40d)))).endObject())
             .get();
         refresh();
@@ -1878,7 +1788,7 @@ public class PercolatorIT extends ESIntegTestCase {
 
         assertAcked(prepareCreate("index").addMapping("mapping", mapping));
         try {
-            client().prepareIndex("index", PercolatorService.TYPE_NAME, "1")
+            client().prepareIndex("index", PercolatorFieldMapper.TYPE_NAME, "1")
                     .setSource(jsonBuilder().startObject().field("query", nestedQuery("nested", matchQuery("nested.name", "value")).innerHit(new QueryInnerHits())).endObject())
                     .execute().actionGet();
             fail("Expected a parse error, because inner_hits isn't supported in the percolate api");
@@ -1893,14 +1803,14 @@ public class PercolatorIT extends ESIntegTestCase {
         // the percolate api
 
         assertAcked(prepareCreate("index").addMapping("child", "_parent", "type=parent").addMapping("parent"));
-        client().prepareIndex("index", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("index", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject().field("query", hasChildQuery("child", matchAllQuery())).endObject())
                 .execute().actionGet();
     }
 
     public void testPercolateDocumentWithParentField() throws Exception {
         assertAcked(prepareCreate("index").addMapping("child", "_parent", "type=parent").addMapping("parent"));
-        client().prepareIndex("index", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("index", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
                 .execute().actionGet();
         refresh();
@@ -1915,7 +1825,7 @@ public class PercolatorIT extends ESIntegTestCase {
     }
 
     public void testFilterByNow() throws Exception {
-        client().prepareIndex("index", PercolatorService.TYPE_NAME, "1")
+        client().prepareIndex("index", PercolatorFieldMapper.TYPE_NAME, "1")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("created", "2015-07-10T14:41:54+0000").endObject())
                 .get();
         refresh();
diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolatorServiceTests.java b/core/src/test/java/org/elasticsearch/percolator/PercolatorServiceTests.java
deleted file mode 100644
index 05a4a156a01..00000000000
--- a/core/src/test/java/org/elasticsearch/percolator/PercolatorServiceTests.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.percolator;
-
-import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.StoredField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.NoMergePolicy;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.memory.MemoryIndex;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.percolate.PercolateShardResponse;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.mapper.ParseContext;
-import org.elasticsearch.index.mapper.Uid;
-import org.elasticsearch.index.mapper.internal.UidFieldMapper;
-import org.elasticsearch.index.percolator.ExtractQueryTermsService;
-import org.elasticsearch.index.percolator.PercolatorFieldMapper;
-import org.elasticsearch.index.percolator.PercolatorQueriesRegistry;
-import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.ContextIndexSearcher;
-import org.elasticsearch.test.ESTestCase;
-import org.junit.After;
-import org.junit.Before;
-
-import java.io.IOException;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class PercolatorServiceTests extends ESTestCase {
-
-    private Directory directory;
-    private IndexWriter indexWriter;
-    private DirectoryReader directoryReader;
-
-    @Before
-    public void init() throws Exception {
-        directory = newDirectory();
-        IndexWriterConfig config = new IndexWriterConfig(new WhitespaceAnalyzer());
-        config.setMergePolicy(NoMergePolicy.INSTANCE);
-        indexWriter = new IndexWriter(directory, config);
-    }
-
-    @After
-    public void destroy() throws Exception {
-        directoryReader.close();
-        directory.close();
-    }
-
-    public void testCount() throws Exception {
-        PercolateContext context = mock(PercolateContext.class);
-        when(context.shardTarget()).thenReturn(new SearchShardTarget("_id", new Index("_index", "_na_"), 0));
-        when(context.percolatorTypeFilter()).thenReturn(new MatchAllDocsQuery());
-        when(context.isOnlyCount()).thenReturn(true);
-        IndexShard shard = mock(IndexShard.class);
-        when(shard.shardId()).thenReturn(new ShardId("_index", "_na_", 0));
-        when(context.indexShard()).thenReturn(shard);
-
-        PercolatorQueriesRegistry registry = createRegistry();
-        addPercolatorQuery("1", new TermQuery(new Term("field", "brown")), indexWriter, registry);
-        addPercolatorQuery("2", new TermQuery(new Term("field", "fox")), indexWriter, registry);
-        addPercolatorQuery("3", new TermQuery(new Term("field", "monkey")), indexWriter, registry);
-
-        indexWriter.close();
-        directoryReader = DirectoryReader.open(directory);
-        IndexSearcher shardSearcher = newSearcher(directoryReader);
-        when(context.searcher()).thenReturn(new ContextIndexSearcher(new Engine.Searcher("test", shardSearcher), shardSearcher.getQueryCache(), shardSearcher.getQueryCachingPolicy()));
-
-        MemoryIndex memoryIndex = new MemoryIndex();
-        memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer());
-        IndexSearcher percolateSearcher = memoryIndex.createSearcher();
-        when(context.docSearcher()).thenReturn(percolateSearcher);
-
-        PercolateShardResponse response = PercolatorService.doPercolate(context, registry, null, null, null);
-        assertThat(response.topDocs().totalHits, equalTo(2));
-    }
-
-    public void testTopMatching() throws Exception {
-        PercolateContext context = mock(PercolateContext.class);
-        when(context.shardTarget()).thenReturn(new SearchShardTarget("_id", new Index("_index", "_na_"), 0));
-        when(context.percolatorTypeFilter()).thenReturn(new MatchAllDocsQuery());
-        when(context.size()).thenReturn(10);
-        IndexShard shard = mock(IndexShard.class);
-        when(shard.shardId()).thenReturn(new ShardId("_index", "_na_", 0));
-        when(context.indexShard()).thenReturn(shard);
-
-        PercolatorQueriesRegistry registry = createRegistry();
-        addPercolatorQuery("1", new TermQuery(new Term("field", "brown")), indexWriter, registry);
-        addPercolatorQuery("2", new TermQuery(new Term("field", "monkey")), indexWriter, registry);
-        addPercolatorQuery("3", new TermQuery(new Term("field", "fox")), indexWriter, registry);
-
-        indexWriter.close();
-        directoryReader = DirectoryReader.open(directory);
-        IndexSearcher shardSearcher = newSearcher(directoryReader);
-        when(context.searcher()).thenReturn(new ContextIndexSearcher(new Engine.Searcher("test", shardSearcher), shardSearcher.getQueryCache(), shardSearcher.getQueryCachingPolicy()));
-
-        MemoryIndex memoryIndex = new MemoryIndex();
-        memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer());
-        IndexSearcher percolateSearcher = memoryIndex.createSearcher();
-        when(context.docSearcher()).thenReturn(percolateSearcher);
-
-        PercolateShardResponse response = PercolatorService.doPercolate(context, registry, null, null, null);
-        TopDocs topDocs = response.topDocs();
-        assertThat(topDocs.totalHits, equalTo(2));
-        assertThat(topDocs.scoreDocs.length, equalTo(2));
-        assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
-        assertThat(topDocs.scoreDocs[1].doc, equalTo(2));
-    }
-
-    void addPercolatorQuery(String id, Query query, IndexWriter writer, PercolatorQueriesRegistry registry) throws IOException {
-        registry.getPercolateQueries().put(new BytesRef(id), query);
-        ParseContext.Document document = new ParseContext.Document();
-        FieldType extractedQueryTermsFieldType = new FieldType();
-        extractedQueryTermsFieldType.setTokenized(false);
-        extractedQueryTermsFieldType.setIndexOptions(IndexOptions.DOCS);
-        extractedQueryTermsFieldType.freeze();
-        ExtractQueryTermsService.extractQueryTerms(query, document, PercolatorFieldMapper.EXTRACTED_TERMS_FULL_FIELD_NAME, PercolatorFieldMapper.UNKNOWN_QUERY_FULL_FIELD_NAME, extractedQueryTermsFieldType);
-        document.add(new StoredField(UidFieldMapper.NAME, Uid.createUid(PercolatorService.TYPE_NAME, id)));
-        writer.addDocument(document);
-    }
-
-    PercolatorQueriesRegistry createRegistry() {
-        Index index = new Index("_index", "_na_");
-        IndexSettings indexSettings = new IndexSettings(new IndexMetaData.Builder("_index").settings(
-                Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
-                        .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
-                .build(), Settings.EMPTY);
-        return new PercolatorQueriesRegistry(
-                new ShardId(index, 0),
-                indexSettings,
-                null
-        );
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/percolator/RecoveryPercolatorIT.java b/core/src/test/java/org/elasticsearch/percolator/RecoveryPercolatorIT.java
deleted file mode 100644
index a08eb41236d..00000000000
--- a/core/src/test/java/org/elasticsearch/percolator/RecoveryPercolatorIT.java
+++ /dev/null
@@ -1,341 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
-import org.elasticsearch.action.percolate.MultiPercolateRequestBuilder;
-import org.elasticsearch.action.percolate.MultiPercolateResponse;
-import org.elasticsearch.action.percolate.PercolateRequestBuilder;
-import org.elasticsearch.action.percolate.PercolateResponse;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.ESIntegTestCase.Scope;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.function.Predicate;
-
-import static org.elasticsearch.action.percolate.PercolateSourceBuilder.docBuilder;
-import static org.elasticsearch.client.Requests.clusterHealthRequest;
-import static org.elasticsearch.common.settings.Settings.settingsBuilder;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
-import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.elasticsearch.percolator.PercolatorTestUtil.convertFromTextArray;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertMatchCount;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
-import static org.hamcrest.Matchers.arrayWithSize;
-import static org.hamcrest.Matchers.emptyArray;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
-
-@ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0)
-public class RecoveryPercolatorIT extends ESIntegTestCase {
-    @Override
-    protected int numberOfShards() {
-        return 1;
-    }
-
-    public void testRestartNodePercolator1() throws Exception {
-        internalCluster().startNode();
-        assertAcked(prepareCreate("test").addMapping("type1", "field1", "type=text").addMapping(PercolatorService.TYPE_NAME, "color", "type=text"));
-
-        logger.info("--> register a query");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "kuku")
-                .setSource(jsonBuilder().startObject()
-                        .field("color", "blue")
-                        .field("query", termQuery("field1", "value1"))
-                        .endObject())
-                .setRefresh(true)
-                .get();
-
-        PercolateResponse percolate = client().preparePercolate()
-                .setIndices("test").setDocumentType("type1")
-                .setSource(jsonBuilder().startObject().startObject("doc")
-                        .field("field1", "value1")
-                        .endObject().endObject())
-                .get();
-        assertThat(percolate.getMatches(), arrayWithSize(1));
-
-        internalCluster().rollingRestart();
-
-        logger.info("Running Cluster Health (wait for the shards to startup)");
-        ensureYellow();
-
-        percolate = client().preparePercolate()
-                .setIndices("test").setDocumentType("type1")
-                .setSource(jsonBuilder().startObject().startObject("doc")
-                        .field("field1", "value1")
-                        .endObject().endObject())
-                .get();
-        assertMatchCount(percolate, 1L);
-        assertThat(percolate.getMatches(), arrayWithSize(1));
-    }
-
-    public void testRestartNodePercolator2() throws Exception {
-        internalCluster().startNode();
-        assertAcked(prepareCreate("test").addMapping("type1", "field1", "type=text").addMapping(PercolatorService.TYPE_NAME, "color", "type=text"));
-
-        logger.info("--> register a query");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "kuku")
-                .setSource(jsonBuilder().startObject()
-                        .field("color", "blue")
-                        .field("query", termQuery("field1", "value1"))
-                        .endObject())
-                .setRefresh(true)
-                .get();
-
-        assertThat(client().prepareSearch().setSize(0).setTypes(PercolatorService.TYPE_NAME).setQuery(matchAllQuery()).get().getHits().totalHits(), equalTo(1L));
-
-        PercolateResponse percolate = client().preparePercolate()
-                .setIndices("test").setDocumentType("type1")
-                .setSource(jsonBuilder().startObject().startObject("doc")
-                        .field("field1", "value1")
-                        .endObject().endObject())
-                .get();
-        assertMatchCount(percolate, 1L);
-        assertThat(percolate.getMatches(), arrayWithSize(1));
-
-        internalCluster().rollingRestart();
-
-        logger.info("Running Cluster Health (wait for the shards to startup)");
-        ClusterHealthResponse clusterHealth = client().admin().cluster().health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1)).actionGet();
-        logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
-        assertThat(clusterHealth.isTimedOut(), equalTo(false));
-        SearchResponse countResponse = client().prepareSearch().setSize(0).setTypes(PercolatorService.TYPE_NAME).setQuery(matchAllQuery()).get();
-        assertHitCount(countResponse, 1L);
-
-        DeleteIndexResponse actionGet = client().admin().indices().prepareDelete("test").get();
-        assertThat(actionGet.isAcknowledged(), equalTo(true));
-        assertAcked(prepareCreate("test").addMapping("type1", "field1", "type=text").addMapping(PercolatorService.TYPE_NAME, "color", "type=text"));
-        clusterHealth = client().admin().cluster().health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1)).actionGet();
-        logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
-        assertThat(clusterHealth.isTimedOut(), equalTo(false));
-        assertThat(client().prepareSearch().setSize(0).setTypes(PercolatorService.TYPE_NAME).setQuery(matchAllQuery()).get().getHits().totalHits(), equalTo(0L));
-
-        percolate = client().preparePercolate()
-                .setIndices("test").setDocumentType("type1")
-                .setSource(jsonBuilder().startObject().startObject("doc")
-                        .field("field1", "value1")
-                        .endObject().endObject())
-                .get();
-        assertMatchCount(percolate, 0L);
-        assertThat(percolate.getMatches(), emptyArray());
-
-        logger.info("--> register a query");
-        client().prepareIndex("test", PercolatorService.TYPE_NAME, "kuku")
-                .setSource(jsonBuilder().startObject()
-                        .field("color", "blue")
-                        .field("query", termQuery("field1", "value1"))
-                        .endObject())
-                .setRefresh(true)
-                .get();
-
-        assertThat(client().prepareSearch().setSize(0).setTypes(PercolatorService.TYPE_NAME).setQuery(matchAllQuery()).get().getHits().totalHits(), equalTo(1L));
-
-        percolate = client().preparePercolate()
-                .setIndices("test").setDocumentType("type1")
-                .setSource(jsonBuilder().startObject().startObject("doc")
-                        .field("field1", "value1")
-                        .endObject().endObject())
-                .get();
-        assertMatchCount(percolate, 1L);
-        assertThat(percolate.getMatches(), arrayWithSize(1));
-    }
-
-    public void testLoadingPercolateQueriesDuringCloseAndOpen() throws Exception {
-        internalCluster().startNode();
-        internalCluster().startNode();
-
-        assertAcked(client().admin().indices().prepareCreate("test")
-                .setSettings(settingsBuilder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2)
-                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)));
-        ensureGreen();
-
-        logger.info("--> Add dummy docs");
-        client().prepareIndex("test", "type1", "1").setSource("field1", 0).get();
-        client().prepareIndex("test", "type2", "1").setSource("field1", 1).get();
-
-        logger.info("--> register a queries");
-        for (int i = 1; i <= 100; i++) {
-            client().prepareIndex("test", PercolatorService.TYPE_NAME, Integer.toString(i))
-                    .setSource(jsonBuilder().startObject()
-                            .field("query", rangeQuery("field1").from(0).to(i))
-                            .endObject())
-                    .get();
-        }
-        refresh();
-
-        logger.info("--> Percolate doc with field1=95");
-        PercolateResponse response = client().preparePercolate()
-                .setIndices("test").setDocumentType("type1")
-                .setSource(jsonBuilder().startObject().startObject("doc").field("field1", 95).endObject().endObject())
-                .get();
-        assertMatchCount(response, 6L);
-        assertThat(response.getMatches(), arrayWithSize(6));
-        assertThat(convertFromTextArray(response.getMatches(), "test"), arrayContainingInAnyOrder("95", "96", "97", "98", "99", "100"));
-
-        logger.info("--> Close and open index to trigger percolate queries loading...");
-        assertAcked(client().admin().indices().prepareClose("test"));
-        assertAcked(client().admin().indices().prepareOpen("test"));
-        ensureGreen();
-
-        logger.info("--> Percolate doc with field1=100");
-        response = client().preparePercolate()
-                .setIndices("test").setDocumentType("type1")
-                .setSource(jsonBuilder().startObject().startObject("doc").field("field1", 100).endObject().endObject()).get();
-
-        assertMatchCount(response, 1L);
-        assertThat(response.getMatches(), arrayWithSize(1));
-        assertThat(response.getMatches()[0].getId().string(), equalTo("100"));
-    }
-
-    public void testPercolatorRecovery() throws Exception {
-        // 3 nodes, 2 primary + 2 replicas per primary, so each node should have a copy of the data.
-        // We only start and stop nodes 2 and 3, so all requests should succeed and never be partial.
-        internalCluster().startNode(settingsBuilder().put("node.stay", true));
-        internalCluster().startNode(settingsBuilder().put("node.stay", false));
-        internalCluster().startNode(settingsBuilder().put("node.stay", false));
-        ensureGreen();
-        client().admin().indices().prepareCreate("test")
-                .setSettings(settingsBuilder()
-                        .put("index.number_of_shards", 2)
-                        .put("index.number_of_replicas", 2)
-                )
-                .get();
-        ensureGreen();
-
-        final Client client = internalCluster().client(input -> input.getAsBoolean("node.stay", true));
-        final int numQueries = randomIntBetween(50, 100);
-        logger.info("--> register a queries");
-        for (int i = 0; i < numQueries; i++) {
-            client.prepareIndex("test", PercolatorService.TYPE_NAME, Integer.toString(i))
-                    .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                    .get();
-        }
-        refresh();
-
-        final String document = "{\"field\" : \"a\"}";
-        client.prepareIndex("test", "type", "1")
-                .setSource(document)
-                .get();
-
-        final Lock lock = new ReentrantLock();
-        final AtomicBoolean run = new AtomicBoolean(true);
-        final AtomicReference<Throwable> error = new AtomicReference<>();
-        Runnable r = () -> {
-            try {
-                while (run.get()) {
-                    PercolateRequestBuilder percolateBuilder = client.preparePercolate()
-                        .setIndices("test").setDocumentType("type").setSize(numQueries);
-                    if (randomBoolean()) {
-                        percolateBuilder.setPercolateDoc(docBuilder().setDoc(document));
-                    } else {
-                        percolateBuilder.setGetRequest(Requests.getRequest("test").type("type").id("1"));
-                    }
-                    PercolateResponse response;
-                    try {
-                        lock.lock();
-                        response = percolateBuilder.get();
-                    } finally {
-                        lock.unlock();
-                    }
-                    assertNoFailures(response);
-                    assertThat(response.getSuccessfulShards(), equalTo(response.getTotalShards()));
-                    assertThat(response.getCount(), equalTo((long) numQueries));
-                    assertThat(response.getMatches().length, equalTo(numQueries));
-                }
-            } catch (Throwable t) {
-                logger.info("Error in percolate thread...", t);
-                run.set(false);
-                error.set(t);
-            }
-        };
-        Thread t = new Thread(r);
-        t.start();
-        Predicate<Settings> nodePredicate = input -> !input.getAsBoolean("node.stay", false);
-        try {
-            // 1 index, 2 primaries, 2 replicas per primary
-            for (int i = 0; i < 4; i++) {
-                try {
-                    lock.lock();
-                    internalCluster().stopRandomNode(nodePredicate);
-                } finally {
-                    lock.unlock();
-                }
-                client.admin().cluster().prepareHealth("test")
-                        .setWaitForEvents(Priority.LANGUID)
-                        .setTimeout(TimeValue.timeValueMinutes(2))
-                        .setWaitForYellowStatus()
-                        .setWaitForActiveShards(4) // 2 nodes, so 4 shards (2 primaries, 2 replicas)
-                        .get();
-                assertThat(error.get(), nullValue());
-                try {
-                    lock.lock();
-                    internalCluster().stopRandomNode(nodePredicate);
-                } finally {
-                    lock.unlock();
-                }
-                client.admin().cluster().prepareHealth("test")
-                        .setWaitForEvents(Priority.LANGUID)
-                        .setTimeout(TimeValue.timeValueMinutes(2))
-                        .setWaitForYellowStatus()
-                        .setWaitForActiveShards(2) // 1 node, so 2 shards (2 primaries, 0 replicas)
-                        .get();
-                assertThat(error.get(), nullValue());
-                internalCluster().startNode(settingsBuilder().put("node.stay", false));
-                client.admin().cluster().prepareHealth("test")
-                        .setWaitForEvents(Priority.LANGUID)
-                        .setTimeout(TimeValue.timeValueMinutes(2))
-                        .setWaitForYellowStatus()
-                        .setWaitForActiveShards(4)  // 2 nodes, so 4 shards (2 primaries, 2 replicas)
-                        .get();
-                assertThat(error.get(), nullValue());
-                internalCluster().startNode(settingsBuilder().put("node.stay", false));
-                client.admin().cluster().prepareHealth("test")
-                        .setWaitForEvents(Priority.LANGUID)
-                        .setTimeout(TimeValue.timeValueMinutes(2))
-                        .setWaitForGreenStatus() // We're confirm the shard settings, so green instead of yellow
-                        .setWaitForActiveShards(6) // 3 nodes, so 6 shards (2 primaries, 4 replicas)
-                        .get();
-                assertThat(error.get(), nullValue());
-            }
-        } finally {
-            run.set(false);
-        }
-        t.join();
-        assertThat(error.get(), nullValue());
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/percolator/TTLPercolatorIT.java b/core/src/test/java/org/elasticsearch/percolator/TTLPercolatorIT.java
deleted file mode 100644
index 52f8ecb4b13..00000000000
--- a/core/src/test/java/org/elasticsearch/percolator/TTLPercolatorIT.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
-import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.percolate.PercolateResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.AlreadyExpiredException;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.hamcrest.Matchers;
-
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-
-import static org.elasticsearch.common.settings.Settings.settingsBuilder;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.percolator.PercolatorTestUtil.convertFromTextArray;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertMatchCount;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.hamcrest.Matchers.arrayContaining;
-import static org.hamcrest.Matchers.emptyArray;
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- */
-@ClusterScope(scope = ESIntegTestCase.Scope.TEST)
-public class TTLPercolatorIT extends ESIntegTestCase {
-    private static final long PURGE_INTERVAL = 200;
-
-    @Override
-    protected void beforeIndexDeletion() {
-    }
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return settingsBuilder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put("indices.ttl.interval", PURGE_INTERVAL, TimeUnit.MILLISECONDS)
-                .build();
-    }
-
-    public void testPercolatingWithTimeToLive() throws Exception {
-        final Client client = client();
-        ensureGreen();
-
-        String percolatorMapping = XContentFactory.jsonBuilder().startObject().startObject(PercolatorService.TYPE_NAME)
-                .startObject("_ttl").field("enabled", true).endObject()
-                .startObject("_timestamp").field("enabled", true).endObject()
-                .endObject().endObject().string();
-
-        String typeMapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
-                .startObject("_ttl").field("enabled", true).endObject()
-                .startObject("_timestamp").field("enabled", true).endObject()
-                .startObject("properties").startObject("field1").field("type", "text").endObject().endObject()
-                .endObject().endObject().string();
-
-        client.admin().indices().prepareCreate("test")
-                .setSettings(settingsBuilder().put("index.number_of_shards", 2))
-                .addMapping(PercolatorService.TYPE_NAME, percolatorMapping)
-                .addMapping("type1", typeMapping)
-                .execute().actionGet();
-        ensureGreen();
-
-        final NumShards test = getNumShards("test");
-
-        long ttl = 1500;
-        long now = System.currentTimeMillis();
-        client.prepareIndex("test", PercolatorService.TYPE_NAME, "kuku").setSource(jsonBuilder()
-                .startObject()
-                .startObject("query")
-                .startObject("term")
-                .field("field1", "value1")
-                .endObject()
-                .endObject()
-                .endObject()
-        ).setRefresh(true).setTTL(ttl).execute().actionGet();
-
-        IndicesStatsResponse response = client.admin().indices().prepareStats("test")
-                .clear().setIndexing(true)
-                .execute().actionGet();
-        assertThat(response.getIndices().get("test").getTotal().getIndexing().getTotal().getIndexCount(), equalTo((long)test.dataCopies));
-
-        PercolateResponse percolateResponse = client.preparePercolate()
-                .setIndices("test").setDocumentType("type1")
-                .setSource(jsonBuilder()
-                        .startObject()
-                        .startObject("doc")
-                        .field("field1", "value1")
-                        .endObject()
-                        .endObject()
-                ).execute().actionGet();
-        assertNoFailures(percolateResponse);
-        if (percolateResponse.getMatches().length == 0) {
-            // OK, ttl + purgeInterval has passed (slow machine or many other tests were running at the same time
-            GetResponse getResponse = client.prepareGet("test", PercolatorService.TYPE_NAME, "kuku").execute().actionGet();
-            assertThat(getResponse.isExists(), equalTo(false));
-            response = client.admin().indices().prepareStats("test")
-                    .clear().setIndexing(true)
-                    .execute().actionGet();
-            long currentDeleteCount = response.getIndices().get("test").getTotal().getIndexing().getTotal().getDeleteCount();
-            assertThat(currentDeleteCount, equalTo((long)test.dataCopies));
-            return;
-        }
-
-        assertThat(convertFromTextArray(percolateResponse.getMatches(), "test"), arrayContaining("kuku"));
-        long timeSpent = System.currentTimeMillis() - now;
-        long waitTime = ttl + PURGE_INTERVAL - timeSpent;
-        if (waitTime >= 0) {
-            Thread.sleep(waitTime); // Doesn't make sense to check the deleteCount before ttl has expired
-        }
-
-        // See comment in SimpleTTLTests
-        logger.info("Checking if the ttl purger has run");
-        assertTrue(awaitBusy(() -> {
-            IndicesStatsResponse indicesStatsResponse = client.admin().indices().prepareStats("test").clear().setIndexing(true).get();
-            // TTL deletes one doc, but it is indexed in the primary shard and replica shards
-            return indicesStatsResponse.getIndices().get("test").getTotal().getIndexing().getTotal().getDeleteCount() == test.dataCopies;
-        }, 5, TimeUnit.SECONDS));
-
-        percolateResponse = client.preparePercolate()
-                .setIndices("test").setDocumentType("type1")
-                .setSource(jsonBuilder()
-                        .startObject()
-                        .startObject("doc")
-                        .field("field1", "value1")
-                        .endObject()
-                        .endObject()
-                ).execute().actionGet();
-        assertMatchCount(percolateResponse, 0L);
-        assertThat(percolateResponse.getMatches(), emptyArray());
-    }
-
-    public void testEnsureTTLDoesNotCreateIndex() throws IOException, InterruptedException {
-        ensureGreen();
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(settingsBuilder()
-                                         .put("indices.ttl.interval", 60, TimeUnit.SECONDS) // 60 sec
-                .build()).get();
-
-        String typeMapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
-                .startObject("_ttl").field("enabled", true).endObject()
-                .endObject().endObject().string();
-
-        client().admin().indices().prepareCreate("test")
-                .setSettings(settingsBuilder().put("index.number_of_shards", 1))
-                .addMapping("type1", typeMapping)
-                .execute().actionGet();
-        ensureGreen();
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(settingsBuilder()
-                                         .put("indices.ttl.interval", 1, TimeUnit.SECONDS)
-                .build()).get();
-
-        for (int i = 0; i < 100; i++) {
-            logger.debug("index doc {} ", i);
-            try {
-                client().prepareIndex("test", "type1", "" + i).setSource(jsonBuilder()
-                        .startObject()
-                        .startObject("query")
-                        .startObject("term")
-                        .field("field1", "value1")
-                        .endObject()
-                        .endObject()
-                        .endObject()
-                ).setTTL(randomIntBetween(1, 500)).setRefresh(true).execute().actionGet();
-            } catch (MapperParsingException e) {
-                logger.info("failed indexing {}", i, e);
-                // if we are unlucky the TTL is so small that we see the expiry date is already in the past when
-                // we parse the doc ignore those...
-                assertThat(e.getCause(), Matchers.instanceOf(AlreadyExpiredException.class));
-            }
-
-        }
-        refresh();
-        assertTrue(awaitBusy(() -> {
-            IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats("test").clear().setIndexing(true).get();
-            logger.debug("delete count [{}]", indicesStatsResponse.getIndices().get("test").getTotal().getIndexing().getTotal().getDeleteCount());
-            // TTL deletes one doc, but it is indexed in the primary shard and replica shards
-            return indicesStatsResponse.getIndices().get("test").getTotal().getIndexing().getTotal().getDeleteCount() != 0;
-        }, 5, TimeUnit.SECONDS));
-        internalCluster().wipeIndices("test");
-        client().admin().indices().prepareCreate("test")
-                .addMapping("type1", typeMapping)
-                .execute().actionGet();
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginCliTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginCliTests.java
deleted file mode 100644
index bd280e4e1d7..00000000000
--- a/core/src/test/java/org/elasticsearch/plugins/PluginCliTests.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.plugins;
-
-import org.elasticsearch.common.cli.CliToolTestCase;
-import org.elasticsearch.common.cli.MockTerminal;
-
-import static org.elasticsearch.common.cli.CliTool.ExitStatus.OK_AND_EXIT;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.is;
-
-public class PluginCliTests extends CliToolTestCase {
-    public void testHelpWorks() throws Exception {
-        MockTerminal terminal = new MockTerminal();
-        assertThat(new PluginCli(terminal).execute(args("--help")), is(OK_AND_EXIT));
-        assertTerminalOutputContainsHelpFile(terminal, "/org/elasticsearch/plugins/plugin.help");
-
-        terminal.resetOutput();
-        assertThat(new PluginCli(terminal).execute(args("install -h")), is(OK_AND_EXIT));
-        assertTerminalOutputContainsHelpFile(terminal, "/org/elasticsearch/plugins/plugin-install.help");
-        for (String plugin : InstallPluginCommand.OFFICIAL_PLUGINS) {
-            assertThat(terminal.getOutput(), containsString(plugin));
-        }
-
-        terminal.resetOutput();
-        assertThat(new PluginCli(terminal).execute(args("remove --help")), is(OK_AND_EXIT));
-        assertTerminalOutputContainsHelpFile(terminal, "/org/elasticsearch/plugins/plugin-remove.help");
-
-        terminal.resetOutput();
-        assertThat(new PluginCli(terminal).execute(args("list -h")), is(OK_AND_EXIT));
-        assertTerminalOutputContainsHelpFile(terminal, "/org/elasticsearch/plugins/plugin-list.help");
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java
index 04bff31057d..73b31b92637 100644
--- a/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java
+++ b/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java
@@ -46,7 +46,6 @@ public class PluginInfoTests extends ESTestCase {
         assertEquals("fake desc", info.getDescription());
         assertEquals("1.0", info.getVersion());
         assertEquals("FakePlugin", info.getClassname());
-        assertTrue(info.isIsolated());
     }
 
     public void testReadFromPropertiesNameMissing() throws Exception {
@@ -203,11 +202,11 @@ public class PluginInfoTests extends ESTestCase {
 
     public void testPluginListSorted() {
         PluginsAndModules pluginsInfo = new PluginsAndModules();
-        pluginsInfo.addPlugin(new PluginInfo("c", "foo", "dummy", "dummyclass", true));
-        pluginsInfo.addPlugin(new PluginInfo("b", "foo", "dummy", "dummyclass", true));
-        pluginsInfo.addPlugin(new PluginInfo("e", "foo", "dummy", "dummyclass", true));
-        pluginsInfo.addPlugin(new PluginInfo("a", "foo", "dummy", "dummyclass", true));
-        pluginsInfo.addPlugin(new PluginInfo("d", "foo", "dummy", "dummyclass", true));
+        pluginsInfo.addPlugin(new PluginInfo("c", "foo", "dummy", "dummyclass"));
+        pluginsInfo.addPlugin(new PluginInfo("b", "foo", "dummy", "dummyclass"));
+        pluginsInfo.addPlugin(new PluginInfo("e", "foo", "dummy", "dummyclass"));
+        pluginsInfo.addPlugin(new PluginInfo("a", "foo", "dummy", "dummyclass"));
+        pluginsInfo.addPlugin(new PluginInfo("d", "foo", "dummy", "dummyclass"));
 
         final List<PluginInfo> infos = pluginsInfo.getPluginInfos();
         List<String> names = infos.stream().map((input) -> input.getName()).collect(Collectors.toList());
diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
index a47217e3048..4b514763f72 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
@@ -165,7 +165,7 @@ public class RecoveriesCollectionTests extends ESSingleNodeTestCase {
 
     long startRecovery(RecoveriesCollection collection, RecoveryTargetService.RecoveryListener listener, TimeValue timeValue) {
         IndicesService indexServices = getInstanceFromNode(IndicesService.class);
-        IndexShard indexShard = indexServices.indexServiceSafe("test").getShardOrNull(0);
+        IndexShard indexShard = indexServices.indexServiceSafe(resolveIndex("test")).getShardOrNull(0);
         final DiscoveryNode sourceNode = new DiscoveryNode("id", DummyTransportAddress.INSTANCE, Version.CURRENT);
         return collection.startRecovery(indexShard, sourceNode, listener, timeValue);
     }
diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java b/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
index ed07b06a2ef..c9ed1921482 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
@@ -22,8 +22,12 @@ package org.elasticsearch.recovery;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
+import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.routing.Murmur3HashFunction;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
@@ -31,6 +35,7 @@ import org.elasticsearch.common.math.MathUtils;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.DocsStats;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.sort.SortOrder;
@@ -50,7 +55,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllS
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout;
 
-@TestLogging("_root:DEBUG")
+@TestLogging("_root:DEBUG,index.shard:TRACE")
 public class RecoveryWhileUnderLoadIT extends ESIntegTestCase {
     private final ESLogger logger = Loggers.getLogger(RecoveryWhileUnderLoadIT.class);
 
@@ -268,14 +273,12 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase {
     private void iterateAssertCount(final int numberOfShards, final long numberOfDocs, final int iterations) throws Exception {
         SearchResponse[] iterationResults = new SearchResponse[iterations];
         boolean error = false;
-        SearchResponse lastErroneousResponse = null;
         for (int i = 0; i < iterations; i++) {
             SearchResponse searchResponse = client().prepareSearch().setSize((int) numberOfDocs).setQuery(matchAllQuery()).addSort("id", SortOrder.ASC).get();
             logSearchResponse(numberOfShards, numberOfDocs, i, searchResponse);
             iterationResults[i] = searchResponse;
             if (searchResponse.getHits().totalHits() != numberOfDocs) {
                 error = true;
-                lastErroneousResponse = searchResponse;
             }
         }
 
@@ -287,12 +290,21 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase {
                 logger.info("shard [{}] - count {}, primary {}", shardStats.getShardRouting().id(), docsStats.getCount(), shardStats.getShardRouting().primary());
             }
 
-
-            for (int doc = 1, hit = 0; hit < lastErroneousResponse.getHits().getHits().length; hit++, doc++) {
-                SearchHit searchHit = lastErroneousResponse.getHits().getAt(hit);
-                while (doc < Integer.parseInt(searchHit.id())) {
-                    logger.info("missing doc [{}], indexed to shard [{}]", doc, MathUtils.mod(Murmur3HashFunction.hash(Integer.toString(doc)), numberOfShards));
-                    doc++;
+            ClusterService clusterService = clusterService();
+            final ClusterState state = clusterService.state();
+            for (int shard = 0; shard < numberOfShards; shard++) {
+                // background indexer starts using ids on 1
+                for (int id = 1; id <= numberOfDocs; id++) {
+                    ShardId docShard = clusterService.operationRouting().shardId(state, "test", Long.toString(id), null);
+                    if (docShard.id() == shard) {
+                        for (ShardRouting shardRouting : state.routingTable().shardRoutingTable("test", shard)) {
+                            GetResponse response = client().prepareGet("test", "type", Long.toString(id))
+                                    .setPreference("_only_node:" + shardRouting.currentNodeId()).get();
+                            if (response.isExists()) {
+                                logger.info("missing id [{}] on shard {}", id, shardRouting);
+                            }
+                        }
+                    }
                 }
             }
 
diff --git a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
index 1fd44959a59..7e455913151 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
@@ -27,13 +27,13 @@ import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
diff --git a/core/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java b/core/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java
index 16f27655055..b441dd32c78 100644
--- a/core/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java
@@ -123,7 +123,7 @@ public class TruncatedRecoveryIT extends ESIntegTestCase {
                 public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException {
                     if (action.equals(RecoveryTargetService.Actions.FILE_CHUNK)) {
                         RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
-                        logger.debug("file chunk [" + req.toString() + "] lastChunk: " + req.lastChunk());
+                        logger.debug("file chunk [{}] lastChunk: {}", req, req.lastChunk());
                         if ((req.name().endsWith("cfs") || req.name().endsWith("fdt")) && req.lastChunk() && truncate.get()) {
                             latch.countDown();
                             throw new RuntimeException("Caused some truncated files for fun and profit");
diff --git a/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
new file mode 100644
index 00000000000..848c62ab2b4
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.cat;
+
+import org.elasticsearch.action.ShardOperationFailedException;
+import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
+import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RestoreSource;
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.Table;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.recovery.RecoveryState;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.elasticsearch.mock.orig.Mockito.when;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.mockito.Mockito.mock;
+
+public class RestRecoveryActionTests extends ESTestCase {
+
+    public void testRestRecoveryAction() {
+        final Settings settings = Settings.EMPTY;
+        final RestController restController = new RestController(settings);
+        final RestRecoveryAction action = new RestRecoveryAction(settings, restController, restController, null);
+        final int totalShards = randomIntBetween(1, 32);
+        final int successfulShards = Math.max(0, totalShards - randomIntBetween(1, 2));
+        final int failedShards = totalShards - successfulShards;
+        final boolean detailed = randomBoolean();
+        final Map<String, List<RecoveryState>> shardRecoveryStates = new HashMap<>();
+        final List<RecoveryState> recoveryStates = new ArrayList<>();
+
+        for (int i = 0; i < successfulShards; i++) {
+            final RecoveryState state = mock(RecoveryState.class);
+            when(state.getShardId()).thenReturn(new ShardId(new Index("index", "_na_"), i));
+            final RecoveryState.Timer timer = mock(RecoveryState.Timer.class);
+            when(timer.time()).thenReturn((long)randomIntBetween(1000000, 10 * 1000000));
+            when(state.getTimer()).thenReturn(timer);
+            when(state.getType()).thenReturn(randomFrom(RecoveryState.Type.values()));
+            when(state.getStage()).thenReturn(randomFrom(RecoveryState.Stage.values()));
+            final DiscoveryNode sourceNode = randomBoolean() ? mock(DiscoveryNode.class) : null;
+            if (sourceNode != null) {
+                when(sourceNode.getHostName()).thenReturn(randomAsciiOfLength(8));
+            }
+            when(state.getSourceNode()).thenReturn(sourceNode);
+            final DiscoveryNode targetNode = mock(DiscoveryNode.class);
+            when(targetNode.getHostName()).thenReturn(randomAsciiOfLength(8));
+            when(state.getTargetNode()).thenReturn(targetNode);
+
+            final RestoreSource restoreSource = randomBoolean() ? mock(RestoreSource.class) : null;
+            if (restoreSource != null) {
+                final SnapshotId snapshotId = mock(SnapshotId.class);
+                when(snapshotId.getRepository()).thenReturn(randomAsciiOfLength(8));
+                when(snapshotId.getSnapshot()).thenReturn(randomAsciiOfLength(8));
+                when(restoreSource.snapshotId()).thenReturn(snapshotId);
+            }
+
+            RecoveryState.Index index = mock(RecoveryState.Index.class);
+
+            final int totalRecoveredFiles = randomIntBetween(1, 64);
+            when(index.totalRecoverFiles()).thenReturn(totalRecoveredFiles);
+            final int recoveredFileCount = randomIntBetween(0, totalRecoveredFiles);
+            when(index.recoveredFileCount()).thenReturn(recoveredFileCount);
+            when(index.recoveredFilesPercent()).thenReturn((100f * recoveredFileCount) / totalRecoveredFiles);
+            when(index.totalFileCount()).thenReturn(randomIntBetween(totalRecoveredFiles, 2 * totalRecoveredFiles));
+
+            final int totalRecoveredBytes = randomIntBetween(1, 1 << 24);
+            when(index.totalRecoverBytes()).thenReturn((long)totalRecoveredBytes);
+            final int recoveredBytes = randomIntBetween(0, totalRecoveredBytes);
+            when(index.recoveredBytes()).thenReturn((long)recoveredBytes);
+            when(index.recoveredBytesPercent()).thenReturn((100f * recoveredBytes) / totalRecoveredBytes);
+            when(index.totalRecoverBytes()).thenReturn((long)randomIntBetween(totalRecoveredBytes, 2 * totalRecoveredBytes));
+            when(state.getIndex()).thenReturn(index);
+
+            final RecoveryState.Translog translog = mock(RecoveryState.Translog.class);
+            final int translogOps = randomIntBetween(0, 1 << 18);
+            when(translog.totalOperations()).thenReturn(translogOps);
+            final int translogOpsRecovered = randomIntBetween(0, translogOps);
+            when(translog.recoveredOperations()).thenReturn(translogOpsRecovered);
+            when(translog.recoveredPercent()).thenReturn(translogOps == 0 ? 100f : (100f * translogOpsRecovered / translogOps));
+            when(state.getTranslog()).thenReturn(translog);
+
+            recoveryStates.add(state);
+        }
+
+        final List<RecoveryState> shuffle = new ArrayList<>(recoveryStates);
+        Randomness.shuffle(shuffle);
+        shardRecoveryStates.put("index", shuffle);
+
+        final List<ShardOperationFailedException> shardFailures = new ArrayList<>();
+        final RecoveryResponse response = new RecoveryResponse(
+                totalShards,
+                successfulShards,
+                failedShards,
+                detailed,
+                shardRecoveryStates,
+                shardFailures);
+        final Table table = action.buildRecoveryTable(null, response);
+
+        assertNotNull(table);
+
+        List<Table.Cell> headers = table.getHeaders();
+        assertThat(headers.get(0).value, equalTo("index"));
+        assertThat(headers.get(1).value, equalTo("shard"));
+        assertThat(headers.get(2).value, equalTo("time"));
+        assertThat(headers.get(3).value, equalTo("type"));
+        assertThat(headers.get(4).value, equalTo("stage"));
+        assertThat(headers.get(5).value, equalTo("source_host"));
+        assertThat(headers.get(6).value, equalTo("target_host"));
+        assertThat(headers.get(7).value, equalTo("repository"));
+        assertThat(headers.get(8).value, equalTo("snapshot"));
+        assertThat(headers.get(9).value, equalTo("files"));
+        assertThat(headers.get(10).value, equalTo("files_recovered"));
+        assertThat(headers.get(11).value, equalTo("files_percent"));
+        assertThat(headers.get(12).value, equalTo("files_total"));
+        assertThat(headers.get(13).value, equalTo("bytes"));
+        assertThat(headers.get(14).value, equalTo("bytes_recovered"));
+        assertThat(headers.get(15).value, equalTo("bytes_percent"));
+        assertThat(headers.get(16).value, equalTo("bytes_total"));
+        assertThat(headers.get(17).value, equalTo("translog_ops"));
+        assertThat(headers.get(18).value, equalTo("translog_ops_recovered"));
+        assertThat(headers.get(19).value, equalTo("translog_ops_percent"));
+
+        assertThat(table.getRows().size(), equalTo(successfulShards));
+        for (int i = 0; i < successfulShards; i++) {
+            final RecoveryState state = recoveryStates.get(i);
+            List<Table.Cell> cells = table.getRows().get(i);
+            assertThat(cells.get(0).value, equalTo("index"));
+            assertThat(cells.get(1).value, equalTo(i));
+            assertThat(cells.get(2).value, equalTo(new TimeValue(state.getTimer().time())));
+            assertThat(cells.get(3).value, equalTo(state.getType().name().toLowerCase(Locale.ROOT)));
+            assertThat(cells.get(4).value, equalTo(state.getStage().name().toLowerCase(Locale.ROOT)));
+            assertThat(cells.get(5).value, equalTo(state.getSourceNode() == null ? "n/a" : state.getSourceNode().getHostName()));
+            assertThat(cells.get(6).value, equalTo(state.getTargetNode().getHostName()));
+            assertThat(
+                    cells.get(7).value,
+                    equalTo(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshotId().getRepository()));
+            assertThat(
+                    cells.get(8).value,
+                    equalTo(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshotId().getSnapshot()));
+            assertThat(cells.get(9).value, equalTo(state.getIndex().totalRecoverFiles()));
+            assertThat(cells.get(10).value, equalTo(state.getIndex().recoveredFileCount()));
+            assertThat(cells.get(11).value, equalTo(percent(state.getIndex().recoveredFilesPercent())));
+            assertThat(cells.get(12).value, equalTo(state.getIndex().totalFileCount()));
+            assertThat(cells.get(13).value, equalTo(state.getIndex().totalRecoverBytes()));
+            assertThat(cells.get(14).value, equalTo(state.getIndex().recoveredBytes()));
+            assertThat(cells.get(15).value, equalTo(percent(state.getIndex().recoveredBytesPercent())));
+            assertThat(cells.get(16).value, equalTo(state.getIndex().totalBytes()));
+            assertThat(cells.get(17).value, equalTo(state.getTranslog().totalOperations()));
+            assertThat(cells.get(18).value, equalTo(state.getTranslog().recoveredOperations()));
+            assertThat(cells.get(19).value, equalTo(percent(state.getTranslog().recoveredPercent())));
+        }
+    }
+
+    private static String percent(float percent) {
+        return String.format(Locale.ROOT, "%1.1f%%", percent);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index 0825da4d4df..a369b44e2b1 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -122,26 +122,21 @@ public class ScriptServiceTests extends ESTestCase {
     }
 
     public void testScriptsWithoutExtensions() throws IOException {
-
         buildScriptService(Settings.EMPTY);
-        logger.info("--> setup two test files one with extension and another without");
         Path testFileNoExt = scriptsFilePath.resolve("test_no_ext");
         Path testFileWithExt = scriptsFilePath.resolve("test_script.tst");
         Streams.copy("test_file_no_ext".getBytes("UTF-8"), Files.newOutputStream(testFileNoExt));
         Streams.copy("test_file".getBytes("UTF-8"), Files.newOutputStream(testFileWithExt));
         resourceWatcherService.notifyNow();
 
-        logger.info("--> verify that file with extension was correctly processed");
         CompiledScript compiledScript = scriptService.compile(new Script("test_script", ScriptType.FILE, "test", null),
                 ScriptContext.Standard.SEARCH, Collections.emptyMap());
         assertThat(compiledScript.compiled(), equalTo((Object) "compiled_test_file"));
 
-        logger.info("--> delete both files");
         Files.delete(testFileNoExt);
         Files.delete(testFileWithExt);
         resourceWatcherService.notifyNow();
 
-        logger.info("--> verify that file with extension was correctly removed");
         try {
             scriptService.compile(new Script("test_script", ScriptType.FILE, "test", null), ScriptContext.Standard.SEARCH,
                     Collections.emptyMap());
@@ -151,6 +146,25 @@ public class ScriptServiceTests extends ESTestCase {
         }
     }
 
+    public void testScriptCompiledOnceHiddenFileDetected() throws IOException {
+        buildScriptService(Settings.EMPTY);
+
+        Path testHiddenFile = scriptsFilePath.resolve(".hidden_file");
+        Streams.copy("test_hidden_file".getBytes("UTF-8"), Files.newOutputStream(testHiddenFile));
+
+        Path testFileScript = scriptsFilePath.resolve("file_script.tst");
+        Streams.copy("test_file_script".getBytes("UTF-8"), Files.newOutputStream(testFileScript));
+        resourceWatcherService.notifyNow();
+
+        CompiledScript compiledScript = scriptService.compile(new Script("file_script", ScriptType.FILE, "test", null),
+                ScriptContext.Standard.SEARCH, Collections.emptyMap());
+        assertThat(compiledScript.compiled(), equalTo((Object) "compiled_test_file_script"));
+
+        Files.delete(testHiddenFile);
+        Files.delete(testFileScript);
+        resourceWatcherService.notifyNow();
+    }
+
     public void testInlineScriptCompiledOnceCache() throws IOException {
         buildScriptService(Settings.EMPTY);
         CompiledScript compiledScript1 = scriptService.compile(new Script("1+1", ScriptType.INLINE, "test", null),
diff --git a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
index b75d9634f69..97bed01369b 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
@@ -48,7 +48,7 @@ public class SearchModuleTests extends ModuleTestCase {
        }
 
        try {
-           module.registerSuggester("term", PhraseSuggester.class);
+           module.registerSuggester("term", PhraseSuggester.PROTOTYPE);
        } catch (IllegalArgumentException e) {
            assertEquals(e.getMessage(), "Can't register the same [suggester] more than once for [term]");
        }
@@ -56,9 +56,9 @@ public class SearchModuleTests extends ModuleTestCase {
 
     public void testRegisterSuggester() {
         SearchModule module = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry());
-        module.registerSuggester("custom", CustomSuggester.class);
+        module.registerSuggester("custom", CustomSuggester.PROTOTYPE);
         try {
-            module.registerSuggester("custom", CustomSuggester.class);
+            module.registerSuggester("custom", CustomSuggester.PROTOTYPE);
         } catch (IllegalArgumentException e) {
             assertEquals(e.getMessage(), "Can't register the same [suggester] more than once for [custom]");
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
index bc6393986b2..4cd69ef604b 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
@@ -20,10 +20,10 @@
 package org.elasticsearch.search.aggregations;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.inject.AbstractModule;
@@ -59,7 +59,6 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.test.cluster.TestClusterService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPoolModule;
 import org.junit.AfterClass;
@@ -75,6 +74,8 @@ import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.containsString;
 
 public class AggregatorParsingTests extends ESTestCase {
@@ -110,8 +111,9 @@ public class AggregatorParsingTests extends ESTestCase {
         namedWriteableRegistry = new NamedWriteableRegistry();
         index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
         Settings indexSettings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        final TestClusterService clusterService = new TestClusterService();
-        clusterService.setState(new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
+        final ThreadPool threadPool = new ThreadPool(settings);
+        final ClusterService clusterService = createClusterService(threadPool);
+        setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
                 .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
         SettingsModule settingsModule = new SettingsModule(settings);
         settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
@@ -146,7 +148,7 @@ public class AggregatorParsingTests extends ESTestCase {
         };
         scriptModule.prepareSettings(settingsModule);
         injector = new ModulesBuilder().add(new EnvironmentModule(new Environment(settings)), settingsModule,
-                new ThreadPoolModule(new ThreadPool(settings)), scriptModule, new IndicesModule() {
+                new ThreadPoolModule(threadPool), scriptModule, new IndicesModule() {
 
                     @Override
                     protected void configure() {
@@ -186,6 +188,7 @@ public class AggregatorParsingTests extends ESTestCase {
 
     @AfterClass
     public static void afterClass() throws Exception {
+        injector.getInstance(ClusterService.class).close();
         terminate(injector.getInstance(ThreadPool.class));
         injector = null;
         index = null;
@@ -196,20 +199,20 @@ public class AggregatorParsingTests extends ESTestCase {
 
     public void testTwoTypes() throws Exception {
         String source = JsonXContent.contentBuilder()
-            .startObject()
-              .startObject("in_stock")
-                  .startObject("filter")
-                      .startObject("range")
-                          .startObject("stock")
-                              .field("gt", 0)
-                          .endObject()
-                      .endObject()
-                  .endObject()
-                  .startObject("terms")
-                      .field("field", "stock")
-                  .endObject()
-              .endObject()
-            .endObject().string();
+                .startObject()
+                .startObject("in_stock")
+                .startObject("filter")
+                .startObject("range")
+                .startObject("stock")
+                .field("gt", 0)
+                .endObject()
+                .endObject()
+                .endObject()
+                .startObject("terms")
+                .field("field", "stock")
+                .endObject()
+                .endObject()
+                .endObject().string();
         try {
             XContentParser parser = XContentFactory.xContent(source).createParser(source);
             QueryParseContext parseContext = new QueryParseContext(queriesRegistry);
@@ -225,27 +228,27 @@ public class AggregatorParsingTests extends ESTestCase {
 
     public void testTwoAggs() throws Exception {
         String source = JsonXContent.contentBuilder()
-          .startObject()
-              .startObject("by_date")
-                  .startObject("date_histogram")
-                      .field("field", "timestamp")
-                      .field("interval", "month")
-                  .endObject()
-                  .startObject("aggs")
-                      .startObject("tag_count")
-                          .startObject("cardinality")
-                              .field("field", "tag")
-                          .endObject()
-                      .endObject()
-                  .endObject()
-                  .startObject("aggs") // 2nd "aggs": illegal
-                      .startObject("tag_count2")
-                          .startObject("cardinality")
-                              .field("field", "tag")
-                          .endObject()
-                      .endObject()
-                  .endObject()
-          .endObject().string();
+                .startObject()
+                .startObject("by_date")
+                .startObject("date_histogram")
+                .field("field", "timestamp")
+                .field("interval", "month")
+                .endObject()
+                .startObject("aggs")
+                .startObject("tag_count")
+                .startObject("cardinality")
+                .field("field", "tag")
+                .endObject()
+                .endObject()
+                .endObject()
+                .startObject("aggs") // 2nd "aggs": illegal
+                .startObject("tag_count2")
+                .startObject("cardinality")
+                .field("field", "tag")
+                .endObject()
+                .endObject()
+                .endObject()
+                .endObject().string();
         try {
             XContentParser parser = XContentFactory.xContent(source).createParser(source);
             QueryParseContext parseContext = new QueryParseContext(queriesRegistry);
@@ -276,16 +279,16 @@ public class AggregatorParsingTests extends ESTestCase {
         }
 
         String source = JsonXContent.contentBuilder()
-          .startObject()
-              .startObject(name)
-                  .startObject("filter")
-                      .startObject("range")
-                          .startObject("stock")
-                              .field("gt", 0)
-                          .endObject()
-                      .endObject()
-                  .endObject()
-          .endObject().string();
+                .startObject()
+                .startObject(name)
+                .startObject("filter")
+                .startObject("range")
+                .startObject("stock")
+                .field("gt", 0)
+                .endObject()
+                .endObject()
+                .endObject()
+                .endObject().string();
         try {
             XContentParser parser = XContentFactory.xContent(source).createParser(source);
             QueryParseContext parseContext = new QueryParseContext(queriesRegistry);
@@ -302,18 +305,18 @@ public class AggregatorParsingTests extends ESTestCase {
     public void testSameAggregationName() throws Exception {
         final String name = randomAsciiOfLengthBetween(1, 10);
         String source = JsonXContent.contentBuilder()
-          .startObject()
-              .startObject(name)
-                  .startObject("terms")
-                      .field("field", "a")
-                  .endObject()
-              .endObject()
-              .startObject(name)
-                  .startObject("terms")
-                      .field("field", "b")
-                  .endObject()
-              .endObject()
-          .endObject().string();
+                .startObject()
+                .startObject(name)
+                .startObject("terms")
+                .field("field", "a")
+                .endObject()
+                .endObject()
+                .startObject(name)
+                .startObject("terms")
+                .field("field", "b")
+                .endObject()
+                .endObject()
+                .endObject().string();
         try {
             XContentParser parser = XContentFactory.xContent(source).createParser(source);
             QueryParseContext parseContext = new QueryParseContext(queriesRegistry);
@@ -329,21 +332,21 @@ public class AggregatorParsingTests extends ESTestCase {
 
     public void testMissingName() throws Exception {
         String source = JsonXContent.contentBuilder()
-          .startObject()
-              .startObject("by_date")
-                  .startObject("date_histogram")
-                      .field("field", "timestamp")
-                      .field("interval", "month")
-                  .endObject()
-                  .startObject("aggs")
-                      // the aggregation name is missing
-                      //.startObject("tag_count")
-                          .startObject("cardinality")
-                              .field("field", "tag")
-                          .endObject()
-                      //.endObject()
-                  .endObject()
-          .endObject().string();
+                .startObject()
+                .startObject("by_date")
+                .startObject("date_histogram")
+                .field("field", "timestamp")
+                .field("interval", "month")
+                .endObject()
+                .startObject("aggs")
+                // the aggregation name is missing
+                //.startObject("tag_count")
+                .startObject("cardinality")
+                .field("field", "tag")
+                .endObject()
+                //.endObject()
+                .endObject()
+                .endObject().string();
         try {
             XContentParser parser = XContentFactory.xContent(source).createParser(source);
             QueryParseContext parseContext = new QueryParseContext(queriesRegistry);
@@ -359,21 +362,21 @@ public class AggregatorParsingTests extends ESTestCase {
 
     public void testMissingType() throws Exception {
         String source = JsonXContent.contentBuilder()
-          .startObject()
-              .startObject("by_date")
-                  .startObject("date_histogram")
-                      .field("field", "timestamp")
-                      .field("interval", "month")
-                  .endObject()
-                  .startObject("aggs")
-                      .startObject("tag_count")
-                          // the aggregation type is missing
-                          //.startObject("cardinality")
-                              .field("field", "tag")
-                          //.endObject()
-                      .endObject()
-                  .endObject()
-          .endObject().string();
+                .startObject()
+                .startObject("by_date")
+                .startObject("date_histogram")
+                .field("field", "timestamp")
+                .field("interval", "month")
+                .endObject()
+                .startObject("aggs")
+                .startObject("tag_count")
+                // the aggregation type is missing
+                //.startObject("cardinality")
+                .field("field", "tag")
+                //.endObject()
+                .endObject()
+                .endObject()
+                .endObject().string();
         try {
             XContentParser parser = XContentFactory.xContent(source).createParser(source);
             QueryParseContext parseContext = new QueryParseContext(queriesRegistry);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index eeffbb73f8b..0d65bb2bf66 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -20,10 +20,10 @@
 package org.elasticsearch.search.aggregations;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Injector;
@@ -60,7 +60,6 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.test.cluster.TestClusterService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPoolModule;
 import org.junit.AfterClass;
@@ -73,6 +72,8 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 
 public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>> extends ESTestCase {
@@ -83,8 +84,8 @@ public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>>
     protected static final String BOOLEAN_FIELD_NAME = "mapped_boolean";
     protected static final String DATE_FIELD_NAME = "mapped_date";
     protected static final String OBJECT_FIELD_NAME = "mapped_object";
-    protected static final String[] mappedFieldNames = new String[] { STRING_FIELD_NAME, INT_FIELD_NAME,
-            DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, OBJECT_FIELD_NAME };
+    protected static final String[] mappedFieldNames = new String[]{STRING_FIELD_NAME, INT_FIELD_NAME,
+            DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, OBJECT_FIELD_NAME};
 
     private static Injector injector;
     private static Index index;
@@ -117,11 +118,12 @@ public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>>
                 .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
                 .build();
 
-        namedWriteableRegistry =  new NamedWriteableRegistry();
+        namedWriteableRegistry = new NamedWriteableRegistry();
         index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
         Settings indexSettings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        final TestClusterService clusterService = new TestClusterService();
-        clusterService.setState(new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
+        final ThreadPool threadPool = new ThreadPool(settings);
+        final ClusterService clusterService = createClusterService(threadPool);
+        setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
                 .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
         SettingsModule settingsModule = new SettingsModule(settings);
         settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
@@ -158,7 +160,7 @@ public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>>
         injector = new ModulesBuilder().add(
                 new EnvironmentModule(new Environment(settings)),
                 settingsModule,
-                new ThreadPoolModule(new ThreadPool(settings)),
+                new ThreadPoolModule(threadPool),
                 scriptModule,
                 new IndicesModule() {
 
@@ -171,6 +173,7 @@ public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>>
                     protected void configureSearch() {
                         // Skip me
                     }
+
                     @Override
                     protected void configureSuggesters() {
                         // Skip me
@@ -200,6 +203,7 @@ public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>>
 
     @AfterClass
     public static void afterClass() throws Exception {
+        injector.getInstance(ClusterService.class).close();
         terminate(injector.getInstance(ThreadPool.class));
         injector = null;
         index = null;
@@ -248,9 +252,8 @@ public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>>
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             testAgg.writeTo(output);
             try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
-                AggregatorBuilder prototype = (AggregatorBuilder) namedWriteableRegistry.getPrototype(AggregatorBuilder.class,
-                    testAgg.getWriteableName());
-                AggregatorBuilder deserializedQuery = prototype.readFrom(in);
+                AggregatorBuilder deserializedQuery = namedWriteableRegistry.getReader(AggregatorBuilder.class, testAgg.getWriteableName())
+                        .read(in);
                 assertEquals(deserializedQuery, testAgg);
                 assertEquals(deserializedQuery.hashCode(), testAgg.hashCode());
                 assertNotSame(deserializedQuery, testAgg);
@@ -290,10 +293,8 @@ public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>>
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             agg.writeTo(output);
             try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
-                AggregatorBuilder prototype = (AggregatorBuilder) namedWriteableRegistry.getPrototype(AggregatorBuilder.class,
-                    agg.getWriteableName());
                 @SuppressWarnings("unchecked")
-                AB secondAgg = (AB) prototype.readFrom(in);
+                AB secondAgg = (AB) namedWriteableRegistry.getReader(AggregatorBuilder.class, agg.getWriteableName()).read(in);
                 return secondAgg;
             }
         }
@@ -309,7 +310,7 @@ public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>>
             }
         } else {
             if (randomBoolean()) {
-                types = new String[] { MetaData.ALL };
+                types = new String[]{MetaData.ALL};
             } else {
                 types = new String[0];
             }
@@ -320,13 +321,13 @@ public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>>
     public String randomNumericField() {
         int randomInt = randomInt(3);
         switch (randomInt) {
-        case 0:
-            return DATE_FIELD_NAME;
-        case 1:
-            return DOUBLE_FIELD_NAME;
-        case 2:
-        default:
-            return INT_FIELD_NAME;
+            case 0:
+                return DATE_FIELD_NAME;
+            case 1:
+                return DOUBLE_FIELD_NAME;
+            case 2:
+            default:
+                return INT_FIELD_NAME;
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index f180ab57162..cac2100ddde 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -20,10 +20,10 @@
 package org.elasticsearch.search.aggregations;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Injector;
@@ -61,7 +61,6 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.test.cluster.TestClusterService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPoolModule;
 import org.junit.AfterClass;
@@ -74,6 +73,8 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 
 public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggregatorBuilder> extends ESTestCase {
@@ -84,8 +85,8 @@ public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggrega
     protected static final String BOOLEAN_FIELD_NAME = "mapped_boolean";
     protected static final String DATE_FIELD_NAME = "mapped_date";
     protected static final String OBJECT_FIELD_NAME = "mapped_object";
-    protected static final String[] mappedFieldNames = new String[] { STRING_FIELD_NAME, INT_FIELD_NAME,
-            DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, OBJECT_FIELD_NAME };
+    protected static final String[] mappedFieldNames = new String[]{STRING_FIELD_NAME, INT_FIELD_NAME,
+            DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, OBJECT_FIELD_NAME};
 
     private static Injector injector;
     private static Index index;
@@ -109,7 +110,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggrega
      */
     @BeforeClass
     public static void init() throws IOException {
-     // we have to prefer CURRENT since with the range of versions we support it's rather unlikely to get the current actually.
+        // we have to prefer CURRENT since with the range of versions we support it's rather unlikely to get the current actually.
         Version version = randomBoolean() ? Version.CURRENT
                 : VersionUtils.randomVersionBetween(random(), Version.V_2_0_0_beta1, Version.CURRENT);
         Settings settings = Settings.settingsBuilder()
@@ -118,11 +119,12 @@ public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggrega
                 .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
                 .build();
 
-        namedWriteableRegistry =  new NamedWriteableRegistry();
+        namedWriteableRegistry = new NamedWriteableRegistry();
         index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
         Settings indexSettings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        final TestClusterService clusterService = new TestClusterService();
-        clusterService.setState(new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
+        final ThreadPool threadPool = new ThreadPool(settings);
+        final ClusterService clusterService = createClusterService(threadPool);
+        setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
                 .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
         SettingsModule settingsModule = new SettingsModule(settings);
         settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
@@ -159,7 +161,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggrega
         injector = new ModulesBuilder().add(
                 new EnvironmentModule(new Environment(settings)),
                 settingsModule,
-                new ThreadPoolModule(new ThreadPool(settings)),
+                new ThreadPoolModule(threadPool),
                 scriptModule,
                 new IndicesModule() {
 
@@ -172,6 +174,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggrega
                     protected void configureSearch() {
                         // Skip me
                     }
+
                     @Override
                     protected void configureSuggesters() {
                         // Skip me
@@ -200,6 +203,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggrega
 
     @AfterClass
     public static void afterClass() throws Exception {
+        injector.getInstance(ClusterService.class).close();
         terminate(injector.getInstance(ThreadPool.class));
         injector = null;
         index = null;
@@ -218,7 +222,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggrega
         AF testAgg = createTestAggregatorFactory();
         AggregatorFactories.Builder factoriesBuilder = AggregatorFactories.builder().skipResolveOrder().addPipelineAggregator(testAgg);
         String contentString = factoriesBuilder.toString();
-        System.out.println(contentString);
+        logger.info("Content string: {}", contentString);
         XContentParser parser = XContentFactory.xContent(contentString).createParser(contentString);
         QueryParseContext parseContext = new QueryParseContext(queriesRegistry);
         parseContext.reset(parser);
@@ -310,7 +314,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggrega
             }
         } else {
             if (randomBoolean()) {
-                types = new String[] { MetaData.ALL };
+                types = new String[]{MetaData.ALL};
             } else {
                 types = new String[0];
             }
@@ -321,13 +325,13 @@ public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggrega
     public String randomNumericField() {
         int randomInt = randomInt(3);
         switch (randomInt) {
-        case 0:
-            return DATE_FIELD_NAME;
-        case 1:
-            return DOUBLE_FIELD_NAME;
-        case 2:
-        default:
-            return INT_FIELD_NAME;
+            case 0:
+                return DATE_FIELD_NAME;
+            case 1:
+                return DOUBLE_FIELD_NAME;
+            case 2:
+            default:
+                return INT_FIELD_NAME;
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java
index 74c308c0fcd..4e4b54d91f6 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java
@@ -185,7 +185,7 @@ children("to_comment", "comment")
         assertThat(categoryTerms.getBuckets().size(), equalTo(3));
 
         for (Terms.Bucket bucket : categoryTerms.getBuckets()) {
-            logger.info("bucket=" + bucket.getKey());
+            logger.info("bucket={}", bucket.getKey());
             Children childrenBucket = bucket.getAggregations().get("to_comment");
             TopHits topHits = childrenBucket.getAggregations().get("top_comments");
             logger.info("total_hits={}", topHits.getHits().getTotalHits());
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
index 21b860ad8cb..afc431f9c68 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
@@ -435,4 +435,27 @@ public class FiltersIT extends ESIntegTestCase {
         assertThat((double) propertiesCounts[2], equalTo((double) sum / numOtherDocs));
     }
 
+    public void testEmptyAggregationWithOtherBucket() throws Exception {
+        SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx")
+                .setQuery(matchAllQuery())
+                .addAggregation(histogram("histo").field("value").interval(1L).minDocCount(0)
+                        .subAggregation(filters("filters", new KeyedFilter("foo", matchAllQuery()))
+                            .otherBucket(true).otherBucketKey("bar")))
+                .execute().actionGet();
+
+        assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L));
+        Histogram histo = searchResponse.getAggregations().get("histo");
+        assertThat(histo, Matchers.notNullValue());
+        Histogram.Bucket bucket = histo.getBuckets().get(1);
+        assertThat(bucket, Matchers.notNullValue());
+
+        Filters filters = bucket.getAggregations().get("filters");
+        assertThat(filters, notNullValue());
+
+        Filters.Bucket other = filters.getBucketByKey("bar");
+        assertThat(other, Matchers.notNullValue());
+        assertThat(other.getKeyAsString(), equalTo("bar"));
+        assertThat(other.getDocCount(), is(0L));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
index 044ca4f8045..11d838d43c4 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
@@ -84,7 +84,7 @@ public class NestedIT extends ESIntegTestCase {
         numParents = randomIntBetween(3, 10);
         numChildren = new int[numParents];
         aggCollectionMode = randomFrom(SubAggCollectionMode.values());
-        logger.info("AGG COLLECTION MODE: " + aggCollectionMode);
+        logger.info("AGG COLLECTION MODE: {}", aggCollectionMode);
         int totalChildren = 0;
         for (int i = 0; i < numParents; ++i) {
             if (i == numParents - 1 && totalChildren == 0) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
index 8304922aa62..3d1e33fa444 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
@@ -433,7 +433,7 @@ public class TopHitsIT extends ESIntegTestCase {
         assertThat(hits.totalHits(), equalTo(controlHits.totalHits()));
         assertThat(hits.getHits().length, equalTo(controlHits.getHits().length));
         for (int i = 0; i < hits.getHits().length; i++) {
-            logger.info(i + ": top_hits: [" + hits.getAt(i).id() + "][" + hits.getAt(i).sortValues()[0] + "] control: [" + controlHits.getAt(i).id() + "][" + controlHits.getAt(i).sortValues()[0] + "]");
+            logger.info("{}: top_hits: [{}][{}] control: [{}][{}]", i, hits.getAt(i).id(), hits.getAt(i).sortValues()[0], controlHits.getAt(i).id(), controlHits.getAt(i).sortValues()[0]);
             assertThat(hits.getAt(i).id(), equalTo(controlHits.getAt(i).id()));
             assertThat(hits.getAt(i).sortValues()[0], equalTo(controlHits.getAt(i).sortValues()[0]));
         }
@@ -609,7 +609,7 @@ public class TopHitsIT extends ESIntegTestCase {
     public void testTrackScores() throws Exception {
         boolean[] trackScores = new boolean[]{true, false};
         for (boolean trackScore : trackScores) {
-            logger.info("Track score=" + trackScore);
+            logger.info("Track score={}", trackScore);
             SearchResponse response = client().prepareSearch("idx").setTypes("field-collapsing")
                     .setQuery(matchQuery("text", "term rare"))
                     .addAggregation(terms("terms")
@@ -618,7 +618,7 @@ public class TopHitsIT extends ESIntegTestCase {
                                             topHits("hits")
                                                     .trackScores(trackScore)
                                                     .size(1)
-                                                    .sort("_id", SortOrder.DESC)
+                                                    .sort("_uid", SortOrder.DESC)
                                     )
                     )
                     .get();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java
index cccac925a1f..05ea6148a56 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java
@@ -29,6 +29,7 @@ import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregatorBuilder;
 import org.elasticsearch.search.fetch.source.FetchSourceContext;
 import org.elasticsearch.search.highlight.HighlightBuilderTests;
+import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
 
@@ -132,7 +133,7 @@ public class TopHitsTests extends BaseAggregationTestCase<TopHitsAggregatorBuild
                     factory.sort(SortBuilders.scoreSort().order(randomFrom(SortOrder.values())));
                     break;
                 case 3:
-                    factory.sort(SortBuilders.scriptSort(new Script("foo"), "number").order(randomFrom(SortOrder.values())));
+                    factory.sort(SortBuilders.scriptSort(new Script("foo"), ScriptSortType.NUMBER).order(randomFrom(SortOrder.values())));
                     break;
                 case 4:
                     factory.sort(randomAsciiOfLengthBetween(5, 20));
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java
index 14e99f0d037..393e1bbfdea 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java
@@ -33,6 +33,7 @@ import org.elasticsearch.search.aggregations.metrics.stats.Stats;
 import org.elasticsearch.search.aggregations.metrics.sum.Sum;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.derivative.Derivative;
+import org.elasticsearch.search.aggregations.pipeline.movavg.models.SimpleModel;
 import org.elasticsearch.search.aggregations.support.AggregationPath;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.hamcrest.Matchers;
@@ -47,7 +48,9 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.filters;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.stats;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.derivative;
+import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.movingAvg;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.closeTo;
@@ -614,6 +617,37 @@ public class DerivativeIT extends ESIntegTestCase {
         }
     }
 
+    public void testAvgMovavgDerivNPE() throws Exception {
+        createIndex("movavg_npe");
+        ensureYellow("movavg_npe");
+
+        for (int i = 0; i < 10; i++) {
+            Integer value = i;
+            if (i == 1 || i == 3) {
+                value = null;
+            }
+
+            XContentBuilder doc = jsonBuilder()
+                    .startObject()
+                    .field("tick", i)
+                    .field("value", value)
+                    .endObject();
+            client().prepareIndex("movavg_npe", "type").setSource(doc).get();
+        }
+
+        refresh();
+
+        SearchResponse response = client()
+                .prepareSearch("movavg_npe")
+                .addAggregation(
+                        histogram("histo").field("tick").interval(1)
+                                .subAggregation(avg("avg").field("value"))
+                                .subAggregation(movingAvg("movavg", "avg").modelBuilder(new SimpleModel.SimpleModelBuilder()).window(3))
+                                .subAggregation(derivative("deriv", "movavg"))).execute().actionGet();
+
+        assertSearchResponse(response);
+    }
+
     private void checkBucketKeyAndDocCount(final String msg, final Histogram.Bucket bucket, final long expectedKey,
             final long expectedDocCount) {
         assertThat(msg, bucket, notNullValue());
diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
index beedd72c280..3168fdc0ff1 100644
--- a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
+++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
@@ -142,7 +142,7 @@ public class SearchWhileRelocatingIT extends ESIntegTestCase {
                 }
                 assertThat("numberOfReplicas: " + numberOfReplicas + " failed in iteration " + i + ", verification: " + verified, thrownExceptions, Matchers.emptyIterable());
                 // if we hit only non-critical exceptions we only make sure that the post search works
-                logger.info("Non-CriticalExceptions: " + nonCriticalExceptions.toString());
+                logger.info("Non-CriticalExceptions: {}", nonCriticalExceptions);
                 assertThat("numberOfReplicas: " + numberOfReplicas + " failed in iteration " + i + ", verification: " + verified, postSearchOK, is(true));
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java
index d342402e4bf..489a604292d 100644
--- a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java
@@ -29,6 +29,7 @@ import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.Settings.Builder;
 import org.elasticsearch.common.settings.SettingsModule;
@@ -152,8 +153,10 @@ public class SearchWithRandomExceptionsIT extends ESIntegTestCase {
     public static class RandomExceptionDirectoryReaderWrapper extends MockEngineSupport.DirectoryReaderWrapper {
 
         public static class TestPlugin extends Plugin {
-            public static final Setting<Double> EXCEPTION_TOP_LEVEL_RATIO_SETTING = Setting.doubleSetting(EXCEPTION_TOP_LEVEL_RATIO_KEY, 0.1d, 0.0d, false, Setting.Scope.INDEX);
-            public static final Setting<Double> EXCEPTION_LOW_LEVEL_RATIO_SETTING = Setting.doubleSetting(EXCEPTION_LOW_LEVEL_RATIO_KEY, 0.1d, 0.0d, false, Setting.Scope.INDEX);
+            public static final Setting<Double> EXCEPTION_TOP_LEVEL_RATIO_SETTING =
+                Setting.doubleSetting(EXCEPTION_TOP_LEVEL_RATIO_KEY, 0.1d, 0.0d, Property.IndexScope);
+            public static final Setting<Double> EXCEPTION_LOW_LEVEL_RATIO_SETTING =
+                Setting.doubleSetting(EXCEPTION_LOW_LEVEL_RATIO_KEY, 0.1d, 0.0d, Property.IndexScope);
             @Override
             public String name() {
                 return "random-exception-reader-wrapper";
diff --git a/core/src/test/java/org/elasticsearch/search/basic/TransportSearchFailuresIT.java b/core/src/test/java/org/elasticsearch/search/basic/TransportSearchFailuresIT.java
index 3cd1d269275..bd1d6ed9795 100644
--- a/core/src/test/java/org/elasticsearch/search/basic/TransportSearchFailuresIT.java
+++ b/core/src/test/java/org/elasticsearch/search/basic/TransportSearchFailuresIT.java
@@ -90,7 +90,7 @@ public class TransportSearchFailuresIT extends ESIntegTestCase {
                 .cluster()
                 .health(clusterHealthRequest("test").waitForYellowStatus().waitForRelocatingShards(0)
                         .waitForActiveShards(test.totalNumShards)).actionGet();
-        logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
+        logger.info("Done Cluster Health, status {}", clusterHealth.getStatus());
         assertThat(clusterHealth.isTimedOut(), equalTo(false));
         assertThat(clusterHealth.getStatus(), anyOf(equalTo(ClusterHealthStatus.YELLOW), equalTo(ClusterHealthStatus.GREEN)));
         assertThat(clusterHealth.getActiveShards(), equalTo(test.totalNumShards));
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index 6b29cabe3f6..74b7b17a037 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -20,10 +20,10 @@
 package org.elasticsearch.search.builder;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -49,15 +49,9 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.AbstractQueryTestCase;
-import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.EmptyQueryBuilder;
-import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryRewriteContext;
-import org.elasticsearch.index.query.TermQueryBuilder;
-import org.elasticsearch.index.query.WrapperQueryBuilder;
-import org.elasticsearch.index.query.functionscore.ScoreFunctionParser;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
@@ -80,15 +74,17 @@ import org.elasticsearch.search.fetch.source.FetchSourceContext;
 import org.elasticsearch.search.highlight.HighlightBuilderTests;
 import org.elasticsearch.search.rescore.QueryRescoreBuilderTests;
 import org.elasticsearch.search.searchafter.SearchAfterBuilder;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.ScoreSortBuilder;
+import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
-import org.elasticsearch.search.suggest.SuggestBuilder;
-import org.elasticsearch.search.suggest.SuggestBuilders;
+import org.elasticsearch.search.suggest.SuggestBuilderTests;
+import org.elasticsearch.search.suggest.Suggesters;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.test.cluster.TestClusterService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPoolModule;
 import org.junit.AfterClass;
@@ -102,6 +98,8 @@ import java.util.List;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 
 public class SearchSourceBuilderTests extends ESTestCase {
@@ -115,6 +113,8 @@ public class SearchSourceBuilderTests extends ESTestCase {
 
     private static AggregatorParsers aggParsers;
 
+    private static Suggesters suggesters;
+
     private static String[] currentTypes;
 
     private static ParseFieldMatcher parseFieldMatcher;
@@ -133,8 +133,9 @@ public class SearchSourceBuilderTests extends ESTestCase {
         namedWriteableRegistry = new NamedWriteableRegistry();
         index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
         Settings indexSettings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        final TestClusterService clusterService = new TestClusterService();
-        clusterService.setState(new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
+        final ThreadPool threadPool = new ThreadPool(settings);
+        final ClusterService clusterService = createClusterService(threadPool);
+        setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
                 .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
         SettingsModule settingsModule = new SettingsModule(settings);
         settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
@@ -170,9 +171,8 @@ public class SearchSourceBuilderTests extends ESTestCase {
         scriptModule.prepareSettings(settingsModule);
         injector = new ModulesBuilder().add(
                 new EnvironmentModule(new Environment(settings)), settingsModule,
-                new ThreadPoolModule(new ThreadPool(settings)),
+                new ThreadPoolModule(threadPool),
                 scriptModule, new IndicesModule() {
-
                     @Override
                     protected void configure() {
                         bindMapperExtension();
@@ -182,13 +182,8 @@ public class SearchSourceBuilderTests extends ESTestCase {
                     protected void configureSearch() {
                         // Skip me
                     }
-                    @Override
-                    protected void configureSuggesters() {
-                        // Skip me
-                    }
                 },
                 new IndexSettingsModule(index, settings),
-
                 new AbstractModule() {
                     @Override
                     protected void configure() {
@@ -199,6 +194,7 @@ public class SearchSourceBuilderTests extends ESTestCase {
                 }
         ).createInjector();
         aggParsers = injector.getInstance(AggregatorParsers.class);
+        suggesters = injector.getInstance(Suggesters.class);
         // create some random type with some default field, those types will
         // stick around for all of the subclasses
         currentTypes = new String[randomIntBetween(0, 5)];
@@ -212,6 +208,7 @@ public class SearchSourceBuilderTests extends ESTestCase {
 
     @AfterClass
     public static void afterClass() throws Exception {
+        injector.getInstance(ClusterService.class).close();
         terminate(injector.getInstance(ThreadPool.class));
         injector = null;
         index = null;
@@ -285,26 +282,26 @@ public class SearchSourceBuilderTests extends ESTestCase {
                 excludes[i] = randomAsciiOfLengthBetween(5, 20);
             }
             switch (branch) {
-            case 0:
-                fetchSourceContext = new FetchSourceContext(randomBoolean());
-                break;
-            case 1:
-                fetchSourceContext = new FetchSourceContext(includes, excludes);
-                break;
-            case 2:
-                fetchSourceContext = new FetchSourceContext(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20));
-                break;
-            case 3:
-                fetchSourceContext = new FetchSourceContext(true, includes, excludes);
-                break;
-            case 4:
-                fetchSourceContext = new FetchSourceContext(includes);
-                break;
-            case 5:
-                fetchSourceContext = new FetchSourceContext(randomAsciiOfLengthBetween(5, 20));
-                break;
-            default:
-                throw new IllegalStateException();
+                case 0:
+                    fetchSourceContext = new FetchSourceContext(randomBoolean());
+                    break;
+                case 1:
+                    fetchSourceContext = new FetchSourceContext(includes, excludes);
+                    break;
+                case 2:
+                    fetchSourceContext = new FetchSourceContext(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20));
+                    break;
+                case 3:
+                    fetchSourceContext = new FetchSourceContext(true, includes, excludes);
+                    break;
+                case 4:
+                    fetchSourceContext = new FetchSourceContext(includes);
+                    break;
+                case 5:
+                    fetchSourceContext = new FetchSourceContext(randomAsciiOfLengthBetween(5, 20));
+                    break;
+                default:
+                    throw new IllegalStateException();
             }
             builder.fetchSource(fetchSourceContext);
         }
@@ -339,25 +336,26 @@ public class SearchSourceBuilderTests extends ESTestCase {
             for (int i = 0; i < numSorts; i++) {
                 int branch = randomInt(5);
                 switch (branch) {
-                case 0:
-                    builder.sort(SortBuilders.fieldSort(randomAsciiOfLengthBetween(5, 20)).order(randomFrom(SortOrder.values())));
-                    break;
-                case 1:
-                    builder.sort(SortBuilders.geoDistanceSort(randomAsciiOfLengthBetween(5, 20),
-                            AbstractQueryTestCase.randomGeohash(1, 12)).order(randomFrom(SortOrder.values())));
-                    break;
-                case 2:
-                    builder.sort(SortBuilders.scoreSort().order(randomFrom(SortOrder.values())));
-                    break;
-                case 3:
-                    builder.sort(SortBuilders.scriptSort(new Script("foo"), "number").order(randomFrom(SortOrder.values())));
-                    break;
-                case 4:
-                    builder.sort(randomAsciiOfLengthBetween(5, 20));
-                    break;
-                case 5:
-                    builder.sort(randomAsciiOfLengthBetween(5, 20), randomFrom(SortOrder.values()));
-                    break;
+                    case 0:
+                        builder.sort(SortBuilders.fieldSort(randomAsciiOfLengthBetween(5, 20)).order(randomFrom(SortOrder.values())));
+                        break;
+                    case 1:
+                        builder.sort(SortBuilders.geoDistanceSort(randomAsciiOfLengthBetween(5, 20),
+                                AbstractQueryTestCase.randomGeohash(1, 12)).order(randomFrom(SortOrder.values())));
+                        break;
+                    case 2:
+                        builder.sort(SortBuilders.scoreSort().order(randomFrom(SortOrder.values())));
+                        break;
+                    case 3:
+                        builder.sort(SortBuilders.scriptSort(new Script("foo"),
+                                ScriptSortType.NUMBER).order(randomFrom(SortOrder.values())));
+                        break;
+                    case 4:
+                        builder.sort(randomAsciiOfLengthBetween(5, 20));
+                        break;
+                    case 5:
+                        builder.sort(randomAsciiOfLengthBetween(5, 20), randomFrom(SortOrder.values()));
+                        break;
                 }
             }
         }
@@ -415,9 +413,7 @@ public class SearchSourceBuilderTests extends ESTestCase {
             builder.highlighter(HighlightBuilderTests.randomHighlighterBuilder());
         }
         if (randomBoolean()) {
-            // NORELEASE need a random suggest builder method
-            builder.suggest(new SuggestBuilder().setText(randomAsciiOfLengthBetween(1, 5)).addSuggestion(
-                    SuggestBuilders.termSuggestion(randomAsciiOfLengthBetween(1, 5))));
+            builder.suggest(SuggestBuilderTests.randomSuggestBuilder());
         }
         if (randomBoolean()) {
             // NORELEASE need a random inner hits builder method
@@ -465,7 +461,7 @@ public class SearchSourceBuilderTests extends ESTestCase {
         if (randomBoolean()) {
             parser.nextToken(); // sometimes we move it on the START_OBJECT to test the embedded case
         }
-        SearchSourceBuilder newBuilder = SearchSourceBuilder.parseSearchSource(parser, parseContext, aggParsers);
+        SearchSourceBuilder newBuilder = SearchSourceBuilder.parseSearchSource(parser, parseContext, aggParsers, suggesters);
         assertNull(parser.nextToken());
         assertEquals(testBuilder, newBuilder);
         assertEquals(testBuilder.hashCode(), newBuilder.hashCode());
@@ -503,14 +499,17 @@ public class SearchSourceBuilderTests extends ESTestCase {
         assertTrue("source builder is not equal to self", secondBuilder.equals(secondBuilder));
         assertTrue("source builder is not equal to its copy", firstBuilder.equals(secondBuilder));
         assertTrue("source builder is not symmetric", secondBuilder.equals(firstBuilder));
-        assertThat("source builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(), equalTo(firstBuilder.hashCode()));
+        assertThat("source builder copy's hashcode is different from original hashcode",
+                secondBuilder.hashCode(), equalTo(firstBuilder.hashCode()));
 
         SearchSourceBuilder thirdBuilder = copyBuilder(secondBuilder);
         assertTrue("source builder is not equal to self", thirdBuilder.equals(thirdBuilder));
         assertTrue("source builder is not equal to its copy", secondBuilder.equals(thirdBuilder));
-        assertThat("source builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(), equalTo(thirdBuilder.hashCode()));
+        assertThat("source builder copy's hashcode is different from original hashcode",
+                secondBuilder.hashCode(), equalTo(thirdBuilder.hashCode()));
         assertTrue("equals is not transitive", firstBuilder.equals(thirdBuilder));
-        assertThat("source builder copy's hashcode is different from original hashcode", firstBuilder.hashCode(), equalTo(thirdBuilder.hashCode()));
+        assertThat("source builder copy's hashcode is different from original hashcode",
+                firstBuilder.hashCode(), equalTo(thirdBuilder.hashCode()));
         assertTrue("equals is not symmetric", thirdBuilder.equals(secondBuilder));
         assertTrue("equals is not symmetric", thirdBuilder.equals(firstBuilder));
     }
@@ -530,16 +529,16 @@ public class SearchSourceBuilderTests extends ESTestCase {
             String restContent = " { \"_source\": { \"includes\": \"include\", \"excludes\": \"*.field2\"}}";
             try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
                 SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.parseSearchSource(parser, createParseContext(parser),
-                        aggParsers);
-                assertArrayEquals(new String[]{"*.field2" }, searchSourceBuilder.fetchSource().excludes());
-                assertArrayEquals(new String[]{"include" }, searchSourceBuilder.fetchSource().includes());
+                        aggParsers, suggesters);
+                assertArrayEquals(new String[]{"*.field2"}, searchSourceBuilder.fetchSource().excludes());
+                assertArrayEquals(new String[]{"include"}, searchSourceBuilder.fetchSource().includes());
             }
         }
         {
             String restContent = " { \"_source\": false}";
             try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
                 SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.parseSearchSource(parser, createParseContext(parser),
-                        aggParsers);
+                        aggParsers, suggesters);
                 assertArrayEquals(new String[]{}, searchSourceBuilder.fetchSource().excludes());
                 assertArrayEquals(new String[]{}, searchSourceBuilder.fetchSource().includes());
                 assertFalse(searchSourceBuilder.fetchSource().fetchSource());
@@ -552,9 +551,9 @@ public class SearchSourceBuilderTests extends ESTestCase {
             String restContent = " { \"sort\": \"foo\"}";
             try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
                 SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.parseSearchSource(parser, createParseContext(parser),
-                        aggParsers);
+                        aggParsers, suggesters);
                 assertEquals(1, searchSourceBuilder.sorts().size());
-                assertEquals("{\"foo\":{}}", searchSourceBuilder.sorts().get(0).toUtf8());
+                assertEquals(new FieldSortBuilder("foo"), searchSourceBuilder.sorts().get(0));
             }
         }
 
@@ -568,13 +567,13 @@ public class SearchSourceBuilderTests extends ESTestCase {
                     "    ]}";
             try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
                 SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.parseSearchSource(parser, createParseContext(parser),
-                        aggParsers);
+                        aggParsers, suggesters);
                 assertEquals(5, searchSourceBuilder.sorts().size());
-                assertEquals("{\"post_date\":{\"order\":\"asc\"}}", searchSourceBuilder.sorts().get(0).toUtf8());
-                assertEquals("\"user\"", searchSourceBuilder.sorts().get(1).toUtf8());
-                assertEquals("{\"name\":\"desc\"}", searchSourceBuilder.sorts().get(2).toUtf8());
-                assertEquals("{\"age\":\"desc\"}", searchSourceBuilder.sorts().get(3).toUtf8());
-                assertEquals("\"_score\"", searchSourceBuilder.sorts().get(4).toUtf8());
+                assertEquals(new FieldSortBuilder("post_date"), searchSourceBuilder.sorts().get(0));
+                assertEquals(new FieldSortBuilder("user"), searchSourceBuilder.sorts().get(1));
+                assertEquals(new FieldSortBuilder("name").order(SortOrder.DESC), searchSourceBuilder.sorts().get(2));
+                assertEquals(new FieldSortBuilder("age").order(SortOrder.DESC), searchSourceBuilder.sorts().get(3));
+                assertEquals(new ScoreSortBuilder(), searchSourceBuilder.sorts().get(4));
             }
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
index 9440a3e91c1..87d5a00d2a2 100644
--- a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
@@ -51,7 +51,6 @@ import org.hamcrest.Matchers;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -94,13 +93,14 @@ import static org.hamcrest.Matchers.notNullValue;
  */
 @ClusterScope(scope = Scope.SUITE)
 public class ChildQuerySearchIT extends ESIntegTestCase {
+
     @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.settingsBuilder().put(super.nodeSettings(nodeOrdinal))
-                // aggressive filter caching so that we can assert on the filter cache size
-                .put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), IndexModule.INDEX_QUERY_CACHE)
-                .put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)
-                .build();
+    public Settings indexSettings() {
+        return Settings.settingsBuilder().put(super.indexSettings())
+            // aggressive filter caching so that we can assert on the filter cache size
+            .put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), IndexModule.INDEX_QUERY_CACHE)
+            .put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)
+            .build();
     }
 
     public void testSelfReferentialIsForbidden() {
@@ -576,20 +576,20 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
                 .setQuery(hasChildQuery("child", termQuery("c_field", "1")).scoreMode(ScoreMode.Max))
                 .get();
         assertHitCount(searchResponse, 1L);
-        assertThat(searchResponse.getHits().getAt(0).explanation().getDescription(), equalTo("Score based on join value p1"));
+        assertThat(searchResponse.getHits().getAt(0).explanation().getDescription(), containsString("join value p1"));
 
         searchResponse = client().prepareSearch("test")
                 .setExplain(true)
                 .setQuery(hasParentQuery("parent", termQuery("p_field", "1")).score(true))
                 .get();
         assertHitCount(searchResponse, 1L);
-        assertThat(searchResponse.getHits().getAt(0).explanation().getDescription(), equalTo("Score based on join value p1"));
+        assertThat(searchResponse.getHits().getAt(0).explanation().getDescription(), containsString("join value p1"));
 
         ExplainResponse explainResponse = client().prepareExplain("test", "parent", parentId)
                 .setQuery(hasChildQuery("child", termQuery("c_field", "1")).scoreMode(ScoreMode.Max))
                 .get();
         assertThat(explainResponse.isExists(), equalTo(true));
-        assertThat(explainResponse.getExplanation().getDetails()[0].getDescription(), equalTo("Score based on join value p1"));
+        assertThat(explainResponse.getExplanation().getDetails()[0].getDescription(), containsString("join value p1"));
     }
 
     List<IndexRequestBuilder> createDocBuilders() {
@@ -1561,7 +1561,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         SearchResponse response;
 
         // Score mode = NONE
-        response = minMaxQuery(ScoreMode.None, 0, 0);
+        response = minMaxQuery(ScoreMode.None, 0, null);
 
         assertThat(response.getHits().totalHits(), equalTo(3L));
         assertThat(response.getHits().hits()[0].id(), equalTo("2"));
@@ -1571,7 +1571,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(response.getHits().hits()[2].id(), equalTo("4"));
         assertThat(response.getHits().hits()[2].score(), equalTo(1f));
 
-        response = minMaxQuery(ScoreMode.None, 1, 0);
+        response = minMaxQuery(ScoreMode.None, 1, null);
 
         assertThat(response.getHits().totalHits(), equalTo(3L));
         assertThat(response.getHits().hits()[0].id(), equalTo("2"));
@@ -1581,7 +1581,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(response.getHits().hits()[2].id(), equalTo("4"));
         assertThat(response.getHits().hits()[2].score(), equalTo(1f));
 
-        response = minMaxQuery(ScoreMode.None, 2, 0);
+        response = minMaxQuery(ScoreMode.None, 2, null);
 
         assertThat(response.getHits().totalHits(), equalTo(2L));
         assertThat(response.getHits().hits()[0].id(), equalTo("3"));
@@ -1589,13 +1589,13 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(response.getHits().hits()[1].id(), equalTo("4"));
         assertThat(response.getHits().hits()[1].score(), equalTo(1f));
 
-        response = minMaxQuery(ScoreMode.None, 3, 0);
+        response = minMaxQuery(ScoreMode.None, 3, null);
 
         assertThat(response.getHits().totalHits(), equalTo(1L));
         assertThat(response.getHits().hits()[0].id(), equalTo("4"));
         assertThat(response.getHits().hits()[0].score(), equalTo(1f));
 
-        response = minMaxQuery(ScoreMode.None, 4, 0);
+        response = minMaxQuery(ScoreMode.None, 4, null);
 
         assertThat(response.getHits().totalHits(), equalTo(0L));
 
@@ -1641,7 +1641,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         }
 
         // Score mode = SUM
-        response = minMaxQuery(ScoreMode.Total, 0, 0);
+        response = minMaxQuery(ScoreMode.Total, 0, null);
 
         assertThat(response.getHits().totalHits(), equalTo(3L));
         assertThat(response.getHits().hits()[0].id(), equalTo("4"));
@@ -1651,7 +1651,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(response.getHits().hits()[2].id(), equalTo("2"));
         assertThat(response.getHits().hits()[2].score(), equalTo(1f));
 
-        response = minMaxQuery(ScoreMode.Total, 1, 0);
+        response = minMaxQuery(ScoreMode.Total, 1, null);
 
         assertThat(response.getHits().totalHits(), equalTo(3L));
         assertThat(response.getHits().hits()[0].id(), equalTo("4"));
@@ -1661,7 +1661,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(response.getHits().hits()[2].id(), equalTo("2"));
         assertThat(response.getHits().hits()[2].score(), equalTo(1f));
 
-        response = minMaxQuery(ScoreMode.Total, 2, 0);
+        response = minMaxQuery(ScoreMode.Total, 2, null);
 
         assertThat(response.getHits().totalHits(), equalTo(2L));
         assertThat(response.getHits().hits()[0].id(), equalTo("4"));
@@ -1669,13 +1669,13 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(response.getHits().hits()[1].id(), equalTo("3"));
         assertThat(response.getHits().hits()[1].score(), equalTo(3f));
 
-        response = minMaxQuery(ScoreMode.Total, 3, 0);
+        response = minMaxQuery(ScoreMode.Total, 3, null);
 
         assertThat(response.getHits().totalHits(), equalTo(1L));
         assertThat(response.getHits().hits()[0].id(), equalTo("4"));
         assertThat(response.getHits().hits()[0].score(), equalTo(6f));
 
-        response = minMaxQuery(ScoreMode.Total, 4, 0);
+        response = minMaxQuery(ScoreMode.Total, 4, null);
 
         assertThat(response.getHits().totalHits(), equalTo(0L));
 
@@ -1721,7 +1721,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         }
 
         // Score mode = MAX
-        response = minMaxQuery(ScoreMode.Max, 0, 0);
+        response = minMaxQuery(ScoreMode.Max, 0, null);
 
         assertThat(response.getHits().totalHits(), equalTo(3L));
         assertThat(response.getHits().hits()[0].id(), equalTo("4"));
@@ -1731,7 +1731,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(response.getHits().hits()[2].id(), equalTo("2"));
         assertThat(response.getHits().hits()[2].score(), equalTo(1f));
 
-        response = minMaxQuery(ScoreMode.Max, 1, 0);
+        response = minMaxQuery(ScoreMode.Max, 1, null);
 
         assertThat(response.getHits().totalHits(), equalTo(3L));
         assertThat(response.getHits().hits()[0].id(), equalTo("4"));
@@ -1741,7 +1741,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(response.getHits().hits()[2].id(), equalTo("2"));
         assertThat(response.getHits().hits()[2].score(), equalTo(1f));
 
-        response = minMaxQuery(ScoreMode.Max, 2, 0);
+        response = minMaxQuery(ScoreMode.Max, 2, null);
 
         assertThat(response.getHits().totalHits(), equalTo(2L));
         assertThat(response.getHits().hits()[0].id(), equalTo("4"));
@@ -1749,13 +1749,13 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(response.getHits().hits()[1].id(), equalTo("3"));
         assertThat(response.getHits().hits()[1].score(), equalTo(2f));
 
-        response = minMaxQuery(ScoreMode.Max, 3, 0);
+        response = minMaxQuery(ScoreMode.Max, 3, null);
 
         assertThat(response.getHits().totalHits(), equalTo(1L));
         assertThat(response.getHits().hits()[0].id(), equalTo("4"));
         assertThat(response.getHits().hits()[0].score(), equalTo(3f));
 
-        response = minMaxQuery(ScoreMode.Max, 4, 0);
+        response = minMaxQuery(ScoreMode.Max, 4, null);
 
         assertThat(response.getHits().totalHits(), equalTo(0L));
 
@@ -1801,7 +1801,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         }
 
         // Score mode = AVG
-        response = minMaxQuery(ScoreMode.Avg, 0, 0);
+        response = minMaxQuery(ScoreMode.Avg, 0, null);
 
         assertThat(response.getHits().totalHits(), equalTo(3L));
         assertThat(response.getHits().hits()[0].id(), equalTo("4"));
@@ -1811,7 +1811,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(response.getHits().hits()[2].id(), equalTo("2"));
         assertThat(response.getHits().hits()[2].score(), equalTo(1f));
 
-        response = minMaxQuery(ScoreMode.Avg, 1, 0);
+        response = minMaxQuery(ScoreMode.Avg, 1, null);
 
         assertThat(response.getHits().totalHits(), equalTo(3L));
         assertThat(response.getHits().hits()[0].id(), equalTo("4"));
@@ -1821,7 +1821,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(response.getHits().hits()[2].id(), equalTo("2"));
         assertThat(response.getHits().hits()[2].score(), equalTo(1f));
 
-        response = minMaxQuery(ScoreMode.Avg, 2, 0);
+        response = minMaxQuery(ScoreMode.Avg, 2, null);
 
         assertThat(response.getHits().totalHits(), equalTo(2L));
         assertThat(response.getHits().hits()[0].id(), equalTo("4"));
@@ -1829,13 +1829,13 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(response.getHits().hits()[1].id(), equalTo("3"));
         assertThat(response.getHits().hits()[1].score(), equalTo(1.5f));
 
-        response = minMaxQuery(ScoreMode.Avg, 3, 0);
+        response = minMaxQuery(ScoreMode.Avg, 3, null);
 
         assertThat(response.getHits().totalHits(), equalTo(1L));
         assertThat(response.getHits().hits()[0].id(), equalTo("4"));
         assertThat(response.getHits().hits()[0].score(), equalTo(2f));
 
-        response = minMaxQuery(ScoreMode.Avg, 4, 0);
+        response = minMaxQuery(ScoreMode.Avg, 4, null);
 
         assertThat(response.getHits().totalHits(), equalTo(0L));
 
diff --git a/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java b/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java
index 0c7c069ec34..39b7bfc6421 100644
--- a/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java
+++ b/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java
@@ -26,10 +26,10 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.MergePolicyConfig;
 import org.elasticsearch.indices.IndicesService;
@@ -65,7 +65,7 @@ public class ParentFieldLoadingIT extends ESIntegTestCase {
         assertAcked(prepareCreate("test")
                 .setSettings(indexSettings)
                 .addMapping("parent")
-                .addMapping("child", childMapping(MappedFieldType.Loading.LAZY)));
+                .addMapping("child", childMapping(false)));
         ensureGreen();
 
         client().prepareIndex("test", "parent", "1").setSource("{}").get();
@@ -90,27 +90,12 @@ public class ParentFieldLoadingIT extends ESIntegTestCase {
         response = client().admin().cluster().prepareClusterStats().get();
         assertThat(response.getIndicesStats().getFieldData().getMemorySizeInBytes(), equalTo(0L));
 
-        logger.info("testing eager loading...");
-        assertAcked(client().admin().indices().prepareDelete("test").get());
-        assertAcked(prepareCreate("test")
-                .setSettings(indexSettings)
-                .addMapping("parent")
-                .addMapping("child", childMapping(MappedFieldType.Loading.EAGER)));
-        ensureGreen();
-
-        client().prepareIndex("test", "parent", "1").setSource("{}").get();
-        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get();
-        refresh();
-
-        response = client().admin().cluster().prepareClusterStats().get();
-        assertThat(response.getIndicesStats().getFieldData().getMemorySizeInBytes(), equalTo(0L));
-
         logger.info("testing eager global ordinals loading...");
         assertAcked(client().admin().indices().prepareDelete("test").get());
         assertAcked(prepareCreate("test")
                 .setSettings(indexSettings)
                 .addMapping("parent")
-                .addMapping("child", childMapping(MappedFieldType.Loading.EAGER_GLOBAL_ORDINALS)));
+                .addMapping("child", childMapping(true)));
         ensureGreen();
 
         // Need to do 2 separate refreshes, otherwise we have 1 segment and then we can't measure if global ordinals
@@ -139,10 +124,11 @@ public class ParentFieldLoadingIT extends ESIntegTestCase {
         assertThat(response.getIndicesStats().getFieldData().getMemorySizeInBytes(), equalTo(0L));
 
         PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping("test").setType("child")
-                .setSource(childMapping(MappedFieldType.Loading.EAGER_GLOBAL_ORDINALS))
+                .setSource(childMapping(true))
                 .setUpdateAllTypes(true)
                 .get();
         assertAcked(putMappingResponse);
+        Index test = resolveIndex("test");
         assertBusy(new Runnable() {
             @Override
             public void run() {
@@ -152,12 +138,12 @@ public class ParentFieldLoadingIT extends ESIntegTestCase {
 
                 boolean verified = false;
                 IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeName);
-                IndexService indexService = indicesService.indexService("test");
+                IndexService indexService = indicesService.indexService(test);
                 if (indexService != null) {
                     MapperService mapperService = indexService.mapperService();
                     DocumentMapper documentMapper = mapperService.documentMapper("child");
                     if (documentMapper != null) {
-                        verified = documentMapper.parentFieldMapper().fieldType().fieldDataType().getLoading() == MappedFieldType.Loading.EAGER_GLOBAL_ORDINALS;
+                        verified = documentMapper.parentFieldMapper().fieldType().eagerGlobalOrdinals();
                     }
                 }
                 assertTrue(verified);
@@ -172,10 +158,10 @@ public class ParentFieldLoadingIT extends ESIntegTestCase {
         assertThat(response.getIndicesStats().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
     }
 
-    private XContentBuilder childMapping(MappedFieldType.Loading loading) throws IOException {
+    private XContentBuilder childMapping(boolean eagerGlobalOrds) throws IOException {
         return jsonBuilder().startObject().startObject("child").startObject("_parent")
                 .field("type", "parent")
-                .startObject("fielddata").field(MappedFieldType.Loading.KEY, loading).endObject()
+                .startObject("fielddata").field("eager_global_ordinals", eagerGlobalOrds).endObject()
                 .endObject().endObject().endObject();
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
index 3159217f174..071f7375383 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
@@ -27,6 +27,7 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.action.termvectors.TermVectorsResponse;
 import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.termvectors.TermVectorsService;
@@ -173,7 +174,7 @@ public class FetchSubPhasePluginIT extends ESIntegTestCase {
                 }
                 hitField.values().add(tv);
             } catch (IOException e) {
-                e.printStackTrace();
+                ESLoggerFactory.getLogger(FetchSubPhasePluginIT.class.getName()).info("Swallowed exception", e);
             }
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
index 8f04bd72756..83e89592b8e 100644
--- a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
+++ b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search.geo;
 
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.locationtech.spatial4j.context.SpatialContext;
 import org.locationtech.spatial4j.distance.DistanceUtils;
 import org.locationtech.spatial4j.exception.InvalidShapeException;
@@ -560,7 +561,7 @@ public class GeoFilterIT extends ESIntegTestCase {
             strategy.makeQuery(args);
             return true;
         } catch (UnsupportedSpatialOperation e) {
-            e.printStackTrace();
+            ESLoggerFactory.getLogger(GeoFilterIT.class.getName()).info("Unsupported spatial operation {}", e, relation);
             return false;
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
index e41e3c178c5..d124fcf6386 100644
--- a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
+++ b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
@@ -299,7 +299,7 @@ public class GeoShapeQueryTests extends ESSingleNodeTestCase {
         // Create a random geometry collection.
         GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(getRandom());
 
-        logger.info("Created Random GeometryCollection containing " + gcb.numShapes() + " shapes");
+        logger.info("Created Random GeometryCollection containing {} shapes", gcb.numShapes());
 
         client().admin().indices().prepareCreate("test").addMapping("type", "location", "type=geo_shape,tree=quadtree")
                 .execute().actionGet();
diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java
index b8f775639f9..bc2c38ef601 100644
--- a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java
@@ -275,7 +275,7 @@ public class HighlightBuilderTests extends ESTestCase {
         Index index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings);
         // shard context will only need indicesQueriesRegistry for building Query objects nested in highlighter
-        QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, indicesQueriesRegistry) {
+        QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, indicesQueriesRegistry, null) {
             @Override
             public MappedFieldType fieldMapper(String name) {
                 TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name);
@@ -400,7 +400,6 @@ public class HighlightBuilderTests extends ESTestCase {
         context.reset(parser);
         highlightBuilder = HighlightBuilder.PROTOTYPE.fromXContent(context);
         assertEquals("expected HighlightBuilder with field", new HighlightBuilder().field(new Field("foo")), highlightBuilder);
-        System.out.println(Math.log(1/(double)(1+1)) + 1.0);
     }
 
     /**
diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
index f34d5b33c9d..dbe2714d05d 100644
--- a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
@@ -2044,7 +2044,7 @@ public class HighlighterSearchIT extends ESIntegTestCase {
                     .query(multiMatchQueryBuilder)
                     .highlighter(highlight().highlightQuery(randomBoolean() ? multiMatchQueryBuilder : null).highlighterType(highlighterType)
                             .field(new Field("field1").requireFieldMatch(true).preTags("<field1>").postTags("</field1>")));
-            logger.info("Running multi-match type: [" + matchQueryType + "] highlight with type: [" + highlighterType + "]");
+            logger.info("Running multi-match type: [{}] highlight with type: [{}]", matchQueryType, highlighterType);
             SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
             assertHitCount(searchResponse, 1L);
             assertHighlight(searchResponse, 0, "field1", 0, anyOf(equalTo("<field1>The quick brown fox</field1> jumps over"),
diff --git a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
index 084e07e0389..d545479c00b 100644
--- a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
@@ -39,7 +39,6 @@ import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.InternalSettingsPlugin;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -1022,7 +1021,7 @@ public class InnerHitsIT extends ESIntegTestCase {
 
         response = client().prepareSearch("index")
                 .setQuery(hasChildQuery("child", matchQuery("field", "value2").queryName("_name2")).innerHit(new QueryInnerHits()))
-                .addSort("_id", SortOrder.ASC)
+                .addSort("_uid", SortOrder.ASC)
                 .get();
         assertHitCount(response, 1);
         assertThat(response.getHits().getAt(0).id(), equalTo("1"));
diff --git a/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java b/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java
index d93d5117274..00fa879f08f 100644
--- a/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java
+++ b/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java
@@ -53,15 +53,17 @@ public class MatchedQueriesIT extends ESIntegTestCase {
         refresh();
 
         SearchResponse searchResponse = client().prepareSearch()
-                .setQuery(boolQuery().must(matchAllQuery()).filter(boolQuery().should(rangeQuery("number").lte(2).queryName("test1")).should(rangeQuery("number").gt(2).queryName("test2")))).get();
+.setQuery(boolQuery().must(matchAllQuery()).filter(boolQuery()
+                        .should(rangeQuery("number").lt(2).queryName("test1")).should(rangeQuery("number").gte(2).queryName("test2"))))
+                .get();
         assertHitCount(searchResponse, 3L);
         for (SearchHit hit : searchResponse.getHits()) {
-            if (hit.id().equals("1") || hit.id().equals("2")) {
-                assertThat(hit.matchedQueries().length, equalTo(1));
-                assertThat(hit.matchedQueries(), hasItemInArray("test1"));
-            } else if (hit.id().equals("3")) {
+            if (hit.id().equals("3") || hit.id().equals("2")) {
                 assertThat(hit.matchedQueries().length, equalTo(1));
                 assertThat(hit.matchedQueries(), hasItemInArray("test2"));
+            } else if (hit.id().equals("1")) {
+                assertThat(hit.matchedQueries().length, equalTo(1));
+                assertThat(hit.matchedQueries(), hasItemInArray("test1"));
             } else {
                 fail("Unexpected document returned with id " + hit.id());
             }
diff --git a/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java b/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
index 8f0ef3c0fbe..651982106c5 100644
--- a/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
+++ b/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
@@ -383,7 +383,7 @@ public class MoreLikeThisIT extends ESIntegTestCase {
         int maxIters = randomIntBetween(10, 20);
         for (int i = 0; i < maxIters; i++) {
             int max_query_terms = randomIntBetween(1, values.length);
-            logger.info("Running More Like This with max_query_terms = %s", max_query_terms);
+            logger.info("Running More Like This with max_query_terms = {}", max_query_terms);
             MoreLikeThisQueryBuilder mltQuery = moreLikeThisQuery(new String[] {"text"}, null, new Item[] {new Item(null, null, "0")})
                     .minTermFreq(1).minDocFreq(1)
                     .maxQueryTerms(max_query_terms).minimumShouldMatch("0%");
@@ -419,7 +419,7 @@ public class MoreLikeThisIT extends ESIntegTestCase {
                     .minTermFreq(1)
                     .minDocFreq(1)
                     .minimumShouldMatch(minimumShouldMatch);
-            logger.info("Testing with minimum_should_match = " + minimumShouldMatch);
+            logger.info("Testing with minimum_should_match = {}", minimumShouldMatch);
             SearchResponse response = client().prepareSearch("test").setTypes("type1")
                     .setQuery(mltQuery).get();
             assertSearchResponse(response);
diff --git a/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java b/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
index 2f932604145..e38ac0ca76e 100644
--- a/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
+++ b/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
@@ -34,6 +34,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.sort.SortBuilders;
+import org.elasticsearch.search.sort.SortMode;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -748,7 +749,7 @@ public class SimpleNestedIT extends ESIntegTestCase {
                 .addSort(
                         SortBuilders.fieldSort("parent.child.child_values")
                                 .setNestedPath("parent.child")
-                                .sortMode("sum")
+                                .sortMode(SortMode.SUM)
                                 .order(SortOrder.ASC)
                 )
                 .execute().actionGet();
@@ -768,7 +769,7 @@ public class SimpleNestedIT extends ESIntegTestCase {
                 .addSort(
                         SortBuilders.fieldSort("parent.child.child_values")
                                 .setNestedPath("parent.child")
-                                .sortMode("sum")
+                                .sortMode(SortMode.SUM)
                                 .order(SortOrder.DESC)
                 )
                 .execute().actionGet();
@@ -789,7 +790,7 @@ public class SimpleNestedIT extends ESIntegTestCase {
                         SortBuilders.fieldSort("parent.child.child_values")
                                 .setNestedPath("parent.child")
                                 .setNestedFilter(QueryBuilders.termQuery("parent.child.filter", true))
-                                .sortMode("sum")
+                                .sortMode(SortMode.SUM)
                                 .order(SortOrder.ASC)
                 )
                 .execute().actionGet();
@@ -809,7 +810,7 @@ public class SimpleNestedIT extends ESIntegTestCase {
                 .addSort(
                         SortBuilders.fieldSort("parent.child.child_values")
                                 .setNestedPath("parent.child")
-                                .sortMode("avg")
+                                .sortMode(SortMode.AVG)
                                 .order(SortOrder.ASC)
                 )
                 .execute().actionGet();
@@ -828,7 +829,7 @@ public class SimpleNestedIT extends ESIntegTestCase {
                 .addSort(
                         SortBuilders.fieldSort("parent.child.child_values")
                                 .setNestedPath("parent.child")
-                                .sortMode("avg")
+                                .sortMode(SortMode.AVG)
                                 .order(SortOrder.DESC)
                 )
                 .execute().actionGet();
@@ -849,7 +850,7 @@ public class SimpleNestedIT extends ESIntegTestCase {
                         SortBuilders.fieldSort("parent.child.child_values")
                                 .setNestedPath("parent.child")
                                 .setNestedFilter(QueryBuilders.termQuery("parent.child.filter", true))
-                                .sortMode("avg")
+                                .sortMode(SortMode.AVG)
                                 .order(SortOrder.ASC)
                 )
                 .execute().actionGet();
diff --git a/core/src/test/java/org/elasticsearch/search/percolator/PercolatorQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/percolator/PercolatorQuerySearchIT.java
new file mode 100644
index 00000000000..fc2a6d02b8b
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/percolator/PercolatorQuerySearchIT.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.percolator;
+
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.index.percolator.PercolatorFieldMapper;
+import org.elasticsearch.index.query.MultiMatchQueryBuilder;
+import org.elasticsearch.search.highlight.HighlightBuilder;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery;
+import static org.elasticsearch.index.query.QueryBuilders.percolatorQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.hamcrest.Matchers.equalTo;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+
+public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
+
+    public void testPercolatorQuery() throws Exception {
+        createIndex("test", client().admin().indices().prepareCreate("test")
+                .addMapping("type", "field1", "type=keyword", "field2", "type=keyword")
+        );
+
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
+            .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
+            .get();
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "2")
+            .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "value")).endObject())
+            .get();
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "3")
+            .setSource(jsonBuilder().startObject().field("query", boolQuery()
+                .must(matchQuery("field1", "value"))
+                .must(matchQuery("field2", "value"))
+            ).endObject()).get();
+        client().admin().indices().prepareRefresh().get();
+
+        BytesReference source = jsonBuilder().startObject().endObject().bytes();
+        logger.info("percolating empty doc");
+        SearchResponse response = client().prepareSearch()
+            .setQuery(percolatorQuery("type", source))
+            .get();
+        assertHitCount(response, 1);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
+
+        source = jsonBuilder().startObject().field("field1", "value").endObject().bytes();
+        logger.info("percolating doc with 1 field");
+        response = client().prepareSearch()
+            .setQuery(percolatorQuery("type", source))
+            .addSort("_uid", SortOrder.ASC)
+            .get();
+        assertHitCount(response, 2);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
+        assertThat(response.getHits().getAt(1).getId(), equalTo("2"));
+
+        source = jsonBuilder().startObject().field("field1", "value").field("field2", "value").endObject().bytes();
+        logger.info("percolating doc with 2 fields");
+        response = client().prepareSearch()
+            .setQuery(percolatorQuery("type", source))
+            .addSort("_uid", SortOrder.ASC)
+            .get();
+        assertHitCount(response, 3);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
+        assertThat(response.getHits().getAt(1).getId(), equalTo("2"));
+        assertThat(response.getHits().getAt(2).getId(), equalTo("3"));
+    }
+
+    public void testPercolatorSpecificQueries()  throws Exception {
+        createIndex("test", client().admin().indices().prepareCreate("test")
+                .addMapping("type", "field1", "type=text", "field2", "type=text")
+        );
+
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
+                .setSource(jsonBuilder().startObject().field("query", commonTermsQuery("field1", "quick brown fox")).endObject())
+                .get();
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "2")
+                .setSource(jsonBuilder().startObject().field("query", multiMatchQuery("quick brown fox", "field1", "field2")
+                        .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)).endObject())
+                .get();
+        client().admin().indices().prepareRefresh().get();
+
+        BytesReference source = jsonBuilder().startObject()
+                .field("field1", "the quick brown fox jumps over the lazy dog")
+                .field("field2", "the quick brown fox falls down into the well")
+                .endObject().bytes();
+        SearchResponse response = client().prepareSearch()
+                .setQuery(percolatorQuery("type", source))
+                .get();
+        assertHitCount(response, 2);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
+        assertThat(response.getHits().getAt(1).getId(), equalTo("2"));
+    }
+
+    public void testPercolatorQueryWithHighlighting() throws Exception {
+        createIndex("test", client().admin().indices().prepareCreate("test")
+                .addMapping("type", "field1", "type=text")
+        );
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "1")
+                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "brown fox")).endObject())
+                .execute().actionGet();
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "2")
+                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "lazy dog")).endObject())
+                .execute().actionGet();
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "3")
+                .setSource(jsonBuilder().startObject().field("query", termQuery("field1", "jumps")).endObject())
+                .execute().actionGet();
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "4")
+                .setSource(jsonBuilder().startObject().field("query", termQuery("field1", "dog")).endObject())
+                .execute().actionGet();
+        client().prepareIndex("test", PercolatorFieldMapper.TYPE_NAME, "5")
+                .setSource(jsonBuilder().startObject().field("query", termQuery("field1", "fox")).endObject())
+                .execute().actionGet();
+        client().admin().indices().prepareRefresh().get();
+
+        BytesReference document = jsonBuilder().startObject()
+                .field("field1", "The quick brown fox jumps over the lazy dog")
+                .endObject().bytes();
+        SearchResponse searchResponse = client().prepareSearch()
+                .setQuery(percolatorQuery("type", document))
+                .highlighter(new HighlightBuilder().field("field1"))
+                .addSort("_uid", SortOrder.ASC)
+                .get();
+        assertHitCount(searchResponse, 5);
+
+        assertThat(searchResponse.getHits().getAt(0).getHighlightFields().get("field1").fragments()[0].string(),
+                equalTo("The quick <em>brown</em> <em>fox</em> jumps over the lazy dog"));
+        assertThat(searchResponse.getHits().getAt(1).getHighlightFields().get("field1").fragments()[0].string(),
+                equalTo("The quick brown fox jumps over the <em>lazy</em> <em>dog</em>"));
+        assertThat(searchResponse.getHits().getAt(2).getHighlightFields().get("field1").fragments()[0].string(),
+                equalTo("The quick brown fox <em>jumps</em> over the lazy dog"));
+        assertThat(searchResponse.getHits().getAt(3).getHighlightFields().get("field1").fragments()[0].string(),
+                equalTo("The quick brown fox jumps over the lazy <em>dog</em>"));
+        assertThat(searchResponse.getHits().getAt(4).getHighlightFields().get("field1").fragments()[0].string(),
+                equalTo("The quick brown <em>fox</em> jumps over the lazy dog"));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/profile/QueryProfilerIT.java b/core/src/test/java/org/elasticsearch/search/profile/QueryProfilerIT.java
index f09b18bdb8a..d9878774576 100644
--- a/core/src/test/java/org/elasticsearch/search/profile/QueryProfilerIT.java
+++ b/core/src/test/java/org/elasticsearch/search/profile/QueryProfilerIT.java
@@ -72,7 +72,7 @@ public class QueryProfilerIT extends ESIntegTestCase {
         int iters = between(20, 100);
         for (int i = 0; i < iters; i++) {
             QueryBuilder q = randomQueryBuilder(stringFields, numericFields, numDocs, 3);
-            logger.info(q.toString());
+            logger.info("Query: {}", q);
 
             SearchResponse resp = client().prepareSearch()
                     .setQuery(q)
@@ -126,13 +126,11 @@ public class QueryProfilerIT extends ESIntegTestCase {
         int iters = between(1, 10);
         for (int i = 0; i < iters; i++) {
             QueryBuilder q = randomQueryBuilder(stringFields, numericFields, numDocs, 3);
-            logger.info(q.toString());
-
+            logger.info("Query: {}", q);
 
             SearchRequestBuilder vanilla = client().prepareSearch("test")
                     .setQuery(q)
                     .setProfile(false)
-                    .addSort("_score", SortOrder.DESC)
                     .addSort("_uid", SortOrder.ASC)
                     .setPreference("_primary")
                     .setSearchType(SearchType.QUERY_THEN_FETCH);
@@ -140,7 +138,6 @@ public class QueryProfilerIT extends ESIntegTestCase {
             SearchRequestBuilder profile = client().prepareSearch("test")
                     .setQuery(q)
                     .setProfile(true)
-                    .addSort("_score", SortOrder.DESC)
                     .addSort("_uid", SortOrder.ASC)
                     .setPreference("_primary")
                     .setSearchType(SearchType.QUERY_THEN_FETCH);
@@ -309,7 +306,7 @@ public class QueryProfilerIT extends ESIntegTestCase {
         refresh();
 
         QueryBuilder q = QueryBuilders.boolQuery();
-        logger.info(q.toString());
+        logger.info("Query: {}", q);
 
         SearchResponse resp = client().prepareSearch()
                 .setQuery(q)
@@ -360,8 +357,7 @@ public class QueryProfilerIT extends ESIntegTestCase {
 
         QueryBuilder q = QueryBuilders.boolQuery().must(QueryBuilders.boolQuery().must(QueryBuilders.boolQuery().must(QueryBuilders.matchQuery("field1", "one"))));
 
-
-        logger.info(q.toString());
+        logger.info("Query: {}", q);
 
         SearchResponse resp = client().prepareSearch()
                 .setQuery(q)
@@ -408,7 +404,7 @@ public class QueryProfilerIT extends ESIntegTestCase {
         QueryBuilder q = QueryBuilders.boostingQuery(QueryBuilders.matchQuery("field1", "one"), QueryBuilders.matchQuery("field1", "two"))
                 .boost(randomFloat())
                 .negativeBoost(randomFloat());
-        logger.info(q.toString());
+        logger.info("Query: {}", q);
 
         SearchResponse resp = client().prepareSearch()
                 .setQuery(q)
@@ -455,7 +451,7 @@ public class QueryProfilerIT extends ESIntegTestCase {
         QueryBuilder q = QueryBuilders.disMaxQuery()
                 .boost(0.33703882f)
                 .add(QueryBuilders.rangeQuery("field2").from(null).to(73).includeLower(true).includeUpper(true));
-        logger.info(q.toString());
+        logger.info("Query: {}", q);
 
         SearchResponse resp = client().prepareSearch()
                 .setQuery(q)
@@ -501,7 +497,7 @@ public class QueryProfilerIT extends ESIntegTestCase {
 
         QueryBuilder q = QueryBuilders.rangeQuery("field2").from(0).to(5);
 
-        logger.info(q.toString());
+        logger.info("Query: {}", q.toString());
 
         SearchResponse resp = client().prepareSearch()
                 .setQuery(q)
@@ -547,7 +543,7 @@ public class QueryProfilerIT extends ESIntegTestCase {
 
         QueryBuilder q = QueryBuilders.matchPhraseQuery("field1", "one two");
 
-        logger.info(q.toString());
+        logger.info("Query: {}", q);
 
         SearchResponse resp = client().prepareSearch()
                 .setQuery(q)
@@ -559,7 +555,7 @@ public class QueryProfilerIT extends ESIntegTestCase {
 
         if (resp.getShardFailures().length > 0) {
             for (ShardSearchFailure f : resp.getShardFailures()) {
-                logger.error(f.toString());
+                logger.error("Shard search failure: {}", f);
             }
             fail();
         }
@@ -603,7 +599,7 @@ public class QueryProfilerIT extends ESIntegTestCase {
         refresh();
         QueryBuilder q = QueryBuilders.rangeQuery("field2").from(0).to(5);
 
-        logger.info(q.toString());
+        logger.info("Query: {}", q);
 
         SearchResponse resp = client().prepareSearch().setQuery(q).setProfile(false).execute().actionGet();
         assertThat("Profile response element should be an empty map", resp.getProfileResults().size(), equalTo(0));
diff --git a/core/src/test/java/org/elasticsearch/search/query/ExistsIT.java b/core/src/test/java/org/elasticsearch/search/query/ExistsIT.java
index 6b226d2c569..54515c6a2b0 100644
--- a/core/src/test/java/org/elasticsearch/search/query/ExistsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/ExistsIT.java
@@ -58,9 +58,6 @@ public class ExistsIT extends ESIntegTestCase {
         XContentBuilder mapping = XContentBuilder.builder(JsonXContent.jsonXContent)
             .startObject()
                 .startObject("type")
-                    .startObject(FieldNamesFieldMapper.NAME)
-                        .field("enabled", randomBoolean())
-                    .endObject()
                     .startObject("properties")
                         .startObject("foo")
                             .field("type", "text")
@@ -89,10 +86,10 @@ public class ExistsIT extends ESIntegTestCase {
             .endObject();
 
         assertAcked(client().admin().indices().prepareCreate("idx").addMapping("type", mapping));
-        @SuppressWarnings("unchecked")
         Map<String, Object> barObject = new HashMap<>();
         barObject.put("foo", "bar");
         barObject.put("bar", singletonMap("bar", "foo"));
+        @SuppressWarnings("unchecked")
         final Map<String, Object>[] sources = new Map[] {
                 // simple property
                 singletonMap("foo", "bar"),
diff --git a/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
index e0bc26c9296..be190b547ea 100644
--- a/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
@@ -156,12 +156,12 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
                 .endObject()
                 .startObject("first_name")
                 .field("type", "text")
-                .field("omit_norms", "true")
+                .field("norms", false)
                 .field("copy_to", "first_name_phrase")
                 .endObject()
                 .startObject("last_name")
                 .field("type", "text")
-                .field("omit_norms", "true")
+                .field("norms", false)
                 .field("copy_to", "last_name_phrase")
                 .endObject()
                 .endObject()
@@ -567,7 +567,7 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
 
         // test if boosts work
         searchResponse = client().prepareSearch("test")
-                .setQuery(randomizeType(multiMatchQuery("the ultimate", "full_name", "first_name", "last_name", "category").field("last_name", 2)
+                .setQuery(randomizeType(multiMatchQuery("the ultimate", "full_name", "first_name", "last_name", "category").field("last_name", 10)
                         .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)
                         .operator(Operator.AND))).get();
         assertFirstHit(searchResponse, hasId("ultimate1"));   // has ultimate in the last_name and that is boosted
diff --git a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
index 44b8636d51a..891911b6202 100644
--- a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -117,7 +117,7 @@ public class SearchQueryIT extends ESIntegTestCase {
     public void testOmitNormsOnAll() throws ExecutionException, InterruptedException, IOException {
         assertAcked(prepareCreate("test")
                 .addMapping("type1", jsonBuilder().startObject().startObject("type1")
-                        .startObject("_all").field("omit_norms", true).endObject()
+                        .startObject("_all").field("norms", false).endObject()
                         .endObject().endObject())
                 .setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)); // only one shard otherwise IDF might be different for comparing scores
 
@@ -1914,6 +1914,7 @@ public class SearchQueryIT extends ESIntegTestCase {
         assertHitCount(client().prepareSearch("test").setSize(0).setQuery(rangeQuery("field").lte(-999999999999L)).get(), 3);
     }
 
+    @AwaitsFix(bugUrl = "NOCOMMIT")
     public void testRangeQueryWithTimeZone() throws Exception {
         assertAcked(prepareCreate("test")
                 .addMapping("type1", "date", "type=date", "num", "type=integer"));
diff --git a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
index f0bb35cc9d1..0386dd847f8 100644
--- a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
@@ -69,7 +69,6 @@ public class QueryRescoreBuilderTests extends ESTestCase {
     @BeforeClass
     public static void init() {
         namedWriteableRegistry = new NamedWriteableRegistry();
-        namedWriteableRegistry.registerPrototype(RescoreBuilder.class, QueryRescorerBuilder.PROTOTYPE);
         indicesQueriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry).buildQueryParserRegistry();
     }
 
@@ -109,14 +108,17 @@ public class QueryRescoreBuilderTests extends ESTestCase {
             assertTrue("rescore builder is not equal to self", secondBuilder.equals(secondBuilder));
             assertTrue("rescore builder is not equal to its copy", firstBuilder.equals(secondBuilder));
             assertTrue("equals is not symmetric", secondBuilder.equals(firstBuilder));
-            assertThat("rescore builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(), equalTo(firstBuilder.hashCode()));
+            assertThat("rescore builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
+                    equalTo(firstBuilder.hashCode()));
 
             RescoreBuilder<?> thirdBuilder = serializedCopy(secondBuilder);
             assertTrue("rescore builder is not equal to self", thirdBuilder.equals(thirdBuilder));
             assertTrue("rescore builder is not equal to its copy", secondBuilder.equals(thirdBuilder));
-            assertThat("rescore builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(), equalTo(thirdBuilder.hashCode()));
+            assertThat("rescore builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
+                    equalTo(thirdBuilder.hashCode()));
             assertTrue("equals is not transitive", firstBuilder.equals(thirdBuilder));
-            assertThat("rescore builder copy's hashcode is different from original hashcode", firstBuilder.hashCode(), equalTo(thirdBuilder.hashCode()));
+            assertThat("rescore builder copy's hashcode is different from original hashcode", firstBuilder.hashCode(),
+                    equalTo(thirdBuilder.hashCode()));
             assertTrue("equals is not symmetric", thirdBuilder.equals(secondBuilder));
             assertTrue("equals is not symmetric", thirdBuilder.equals(firstBuilder));
         }
@@ -159,7 +161,8 @@ public class QueryRescoreBuilderTests extends ESTestCase {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), indexSettings);
         // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer
-        QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, indicesQueriesRegistry) {
+        QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, indicesQueriesRegistry,
+                null) {
             @Override
             public MappedFieldType fieldMapper(String name) {
                 TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name);
@@ -172,7 +175,8 @@ public class QueryRescoreBuilderTests extends ESTestCase {
             QueryRescoreContext rescoreContext = rescoreBuilder.build(mockShardContext);
             XContentParser parser = createParser(rescoreBuilder);
 
-            QueryRescoreContext parsedRescoreContext = (QueryRescoreContext) new RescoreParseElement().parseSingleRescoreContext(parser, mockShardContext);
+            QueryRescoreContext parsedRescoreContext = (QueryRescoreContext) new RescoreParseElement().parseSingleRescoreContext(parser,
+                    mockShardContext);
             assertNotSame(rescoreContext, parsedRescoreContext);
             assertEquals(rescoreContext.window(), parsedRescoreContext.window());
             assertEquals(rescoreContext.query(), parsedRescoreContext.query());
@@ -315,7 +319,8 @@ public class QueryRescoreBuilderTests extends ESTestCase {
      * create random shape that is put under test
      */
     public static org.elasticsearch.search.rescore.QueryRescorerBuilder randomRescoreBuilder() {
-        QueryBuilder<MatchAllQueryBuilder> queryBuilder = new MatchAllQueryBuilder().boost(randomFloat()).queryName(randomAsciiOfLength(20));
+        QueryBuilder<MatchAllQueryBuilder> queryBuilder = new MatchAllQueryBuilder().boost(randomFloat())
+                .queryName(randomAsciiOfLength(20));
         org.elasticsearch.search.rescore.QueryRescorerBuilder rescorer = new
                 org.elasticsearch.search.rescore.QueryRescorerBuilder(queryBuilder);
         if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java b/core/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
index 292f9a495dc..6c0b9963940 100644
--- a/core/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
+++ b/core/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
@@ -282,8 +282,8 @@ public class DuelScrollIT extends ESIntegTestCase {
             }
             assertEquals(control.getHits().getTotalHits(), scrollDocs);
         } catch (AssertionError e) {
-            logger.info("Control:\n" + control);
-            logger.info("Scroll size=" + size + ", from=" + scrollDocs + ":\n" + scroll);
+            logger.info("Control:\n{}", control);
+            logger.info("Scroll size={}, from={}:\n{}", size, scrollDocs, scroll);
             throw e;
         } finally {
             clearScroll(scroll.getScrollId());
diff --git a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
index 6c10a1c8aef..136c1fba2e0 100644
--- a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
@@ -29,7 +29,6 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.internal.DefaultSearchContext;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.ArrayList;
@@ -270,7 +269,7 @@ public class SimpleSearchIT extends ESIntegTestCase {
             searchResponse = client().prepareSearch("test")
                     .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(max))
                     .setTerminateAfter(i).execute().actionGet();
-            assertHitCount(searchResponse, (long)i);
+            assertHitCount(searchResponse, i);
             assertTrue(searchResponse.isTerminatedEarly());
         }
 
diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
index dfea1a9316b..7cc44f22a2f 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
@@ -19,8 +19,9 @@
 
 package org.elasticsearch.search.sort;
 
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.Accountable;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -31,29 +32,74 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
+import org.elasticsearch.index.mapper.ContentPath;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper.BuilderContext;
+import org.elasticsearch.index.mapper.core.DoubleFieldMapper.DoubleFieldType;
+import org.elasticsearch.index.mapper.object.ObjectMapper;
+import org.elasticsearch.index.mapper.object.ObjectMapper.Nested;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.ScriptContextRegistry;
+import org.elasticsearch.script.ScriptEngineRegistry;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptServiceTests.TestEngineService;
+import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.watcher.ResourceWatcherService;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
-public abstract class AbstractSortTestCase<T extends NamedWriteable<T> & ToXContent & SortElementParserTemp<T>> extends ESTestCase {
+public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends ESTestCase {
 
     protected static NamedWriteableRegistry namedWriteableRegistry;
 
     private static final int NUMBER_OF_TESTBUILDERS = 20;
     static IndicesQueriesRegistry indicesQueriesRegistry;
+    private static ScriptService scriptService;
 
     @BeforeClass
-    public static void init() {
+    public static void init() throws IOException {
+        Path genericConfigFolder = createTempDir();
+        Settings baseSettings = Settings.builder()
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .put(Environment.PATH_CONF_SETTING.getKey(), genericConfigFolder)
+                .build();
+        Environment environment = new Environment(baseSettings);
+        ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry
+                .ScriptEngineRegistration(TestEngineService.class, TestEngineService.TYPES)));
+        ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
+        scriptService = new ScriptService(baseSettings, environment, Collections.singleton(new TestEngineService()),
+                new ResourceWatcherService(baseSettings, null), scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
+            @Override
+            public CompiledScript compile(Script script, ScriptContext scriptContext, Map<String, String> params) {
+                return new CompiledScript(ScriptType.INLINE, "mockName", "test", script);
+            }
+        };
+
         namedWriteableRegistry = new NamedWriteableRegistry();
-        namedWriteableRegistry.registerPrototype(GeoDistanceSortBuilder.class, GeoDistanceSortBuilder.PROTOTYPE);
         indicesQueriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry).buildQueryParserRegistry();
     }
 
@@ -68,6 +114,9 @@ public abstract class AbstractSortTestCase<T extends NamedWriteable<T> & ToXCont
     /** Returns mutated version of original so the returned sort is different in terms of equals/hashcode */
     protected abstract T mutate(T original) throws IOException;
 
+    /** Parse the sort from xContent. Just delegate to the SortBuilder's static fromXContent method. */
+    protected abstract T fromXContent(QueryParseContext context, String fieldName) throws IOException;
+
     /**
      * Test that creates new sort from a random test sort and checks both for equality
      */
@@ -79,15 +128,12 @@ public abstract class AbstractSortTestCase<T extends NamedWriteable<T> & ToXCont
             if (randomBoolean()) {
                 builder.prettyPrint();
             }
-            builder.startObject();
             testItem.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            builder.endObject();
-
             XContentParser itemParser = XContentHelper.createParser(builder.bytes());
             itemParser.nextToken();
-            
+
             /*
-             * filter out name of sort, or field name to sort on for element fieldSort 
+             * filter out name of sort, or field name to sort on for element fieldSort
              */
             itemParser.nextToken();
             String elementName = itemParser.currentName();
@@ -95,13 +141,28 @@ public abstract class AbstractSortTestCase<T extends NamedWriteable<T> & ToXCont
 
             QueryParseContext context = new QueryParseContext(indicesQueriesRegistry);
             context.reset(itemParser);
-            NamedWriteable<T> parsedItem = testItem.fromXContent(context, elementName);
+            T parsedItem = fromXContent(context, elementName);
             assertNotSame(testItem, parsedItem);
             assertEquals(testItem, parsedItem);
             assertEquals(testItem.hashCode(), parsedItem.hashCode());
         }
     }
 
+    /**
+     * test that build() outputs a {@link SortField} that is similar to the one
+     * we would get when parsing the xContent the sort builder is rendering out
+     */
+    public void testBuildSortField() throws IOException {
+        QueryShardContext mockShardContext = createMockShardContext();
+        for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
+            T sortBuilder = createTestItem();
+            SortField sortField = sortBuilder.build(mockShardContext);
+            sortFieldAssertions(sortBuilder, sortField);
+        }
+    }
+
+    protected abstract void sortFieldAssertions(T builder, SortField sortField) throws IOException;
+
     /**
      * Test serialization and deserialization of the test sort.
      */
@@ -146,17 +207,55 @@ public abstract class AbstractSortTestCase<T extends NamedWriteable<T> & ToXCont
         }
     }
 
-    protected T copyItem(T original) throws IOException {
+    private QueryShardContext createMockShardContext() {
+        Index index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, Settings.EMPTY);
+        IndicesFieldDataCache cache = new IndicesFieldDataCache(Settings.EMPTY, null);
+        IndexFieldDataService ifds = new IndexFieldDataService(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY),
+                cache, null, null);
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, new BitsetFilterCache.Listener() {
+
+            @Override
+            public void onRemoval(ShardId shardId, Accountable accountable) {
+            }
+
+            @Override
+            public void onCache(ShardId shardId, Accountable accountable) {
+            }
+        });
+        return new QueryShardContext(idxSettings, bitsetFilterCache, ifds, null, null, scriptService,
+                indicesQueriesRegistry, null) {
+            @Override
+            public MappedFieldType fieldMapper(String name) {
+                return provideMappedFieldType(name);
+            }
+
+            @Override
+            public ObjectMapper getObjectMapper(String name) {
+                BuilderContext context = new BuilderContext(Settings.EMPTY, new ContentPath());
+                return new ObjectMapper.Builder<>(name).nested(Nested.newNested(false, false)).build(context);
+            }
+        };
+    }
+
+    /**
+     * Return a field type. We use {@link DoubleFieldType} by default since it is compatible with all sort modes
+     * Tests that require other field type than double can override this.
+     */
+    protected MappedFieldType provideMappedFieldType(String name) {
+        DoubleFieldType doubleFieldType = new DoubleFieldType();
+        doubleFieldType.setName(name);
+        doubleFieldType.setHasDocValues(true);
+        return doubleFieldType;
+    }
+
+    @SuppressWarnings("unchecked")
+    private T copyItem(T original) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             original.writeTo(output);
             try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
-                @SuppressWarnings("unchecked")
-                T prototype = (T) namedWriteableRegistry.getPrototype(getPrototype(), original.getWriteableName());
-                T copy = (T) prototype.readFrom(in);
-                return copy;
+                return (T) namedWriteableRegistry.getReader(SortBuilder.class, original.getWriteableName()).read(in);
             }
         }
     }
-    
-    protected abstract Class<T> getPrototype();
 }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
new file mode 100644
index 00000000000..0d6e0f16bea
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
@@ -0,0 +1,112 @@
+/*
+x * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.sort;
+
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.index.query.QueryParseContext;
+
+import java.io.IOException;
+
+public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder> {
+
+    @Override
+    protected FieldSortBuilder createTestItem() {
+        return randomFieldSortBuilder();
+    }
+
+    public static FieldSortBuilder randomFieldSortBuilder() {
+        String fieldName = rarely() ? FieldSortBuilder.DOC_FIELD_NAME : randomAsciiOfLengthBetween(1, 10);
+        FieldSortBuilder builder = new FieldSortBuilder(fieldName);
+        if (randomBoolean()) {
+            builder.order(RandomSortDataGenerator.order(null));
+        }
+
+        if (randomBoolean()) {
+            builder.missing(RandomSortDataGenerator.missing(builder.missing()));
+        }
+
+        if (randomBoolean()) {
+            builder.unmappedType(RandomSortDataGenerator.randomAscii(builder.unmappedType()));
+        }
+
+        if (randomBoolean()) {
+            builder.sortMode(RandomSortDataGenerator.mode(builder.sortMode()));
+        }
+
+        if (randomBoolean()) {
+            builder.setNestedFilter(RandomSortDataGenerator.nestedFilter(builder.getNestedFilter()));
+        }
+
+        if (randomBoolean()) {
+            builder.setNestedPath(RandomSortDataGenerator.randomAscii(builder.getNestedPath()));
+        }
+
+        return builder;
+    }
+
+    @Override
+    protected FieldSortBuilder mutate(FieldSortBuilder original) throws IOException {
+        FieldSortBuilder mutated = new FieldSortBuilder(original);
+        int parameter = randomIntBetween(0, 5);
+        switch (parameter) {
+        case 0:
+            mutated.setNestedPath(RandomSortDataGenerator.randomAscii(mutated.getNestedPath()));
+            break;
+        case 1:
+            mutated.setNestedFilter(RandomSortDataGenerator.nestedFilter(mutated.getNestedFilter()));
+            break;
+        case 2:
+            mutated.sortMode(RandomSortDataGenerator.mode(mutated.sortMode()));
+            break;
+        case 3:
+            mutated.unmappedType(RandomSortDataGenerator.randomAscii(mutated.unmappedType()));
+            break;
+        case 4:
+            mutated.missing(RandomSortDataGenerator.missing(mutated.missing()));
+            break;
+        case 5:
+            mutated.order(RandomSortDataGenerator.order(mutated.order()));
+            break;
+        default:
+            throw new IllegalStateException("Unsupported mutation.");
+        }
+        return mutated;
+    }
+
+    @Override
+    protected void sortFieldAssertions(FieldSortBuilder builder, SortField sortField) throws IOException {
+        SortField.Type expectedType;
+        if (builder.getFieldName().equals(FieldSortBuilder.DOC_FIELD_NAME)) {
+            expectedType = SortField.Type.DOC;
+        } else {
+            expectedType = SortField.Type.CUSTOM;
+        }
+        assertEquals(expectedType, sortField.getType());
+        assertEquals(builder.order() == SortOrder.ASC ? false : true, sortField.getReverse());
+        if (expectedType == SortField.Type.CUSTOM) {
+            assertEquals(builder.getFieldName(), sortField.getField());
+        }
+    }
+
+    @Override
+    protected FieldSortBuilder fromXContent(QueryParseContext context, String fieldName) throws IOException {
+        return FieldSortBuilder.fromXContent(context, fieldName);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
index 894f0fbe6b9..2419dd2c508 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
@@ -19,44 +19,6 @@
 
 package org.elasticsearch.search.sort;
 
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFirstHit;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertOrderedSearchHits;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSecondHit;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
-import static org.hamcrest.Matchers.closeTo;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.lessThan;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Random;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.Map.Entry;
-import java.util.concurrent.ExecutionException;
-
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
@@ -80,6 +42,43 @@ import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.hamcrest.Matchers;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ExecutionException;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFirstHit;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSecondHit;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
 public class FieldSortIT extends ESIntegTestCase {
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
@@ -480,18 +479,11 @@ public class FieldSortIT extends ESIntegTestCase {
         assertAcked(prepareCreate("test")
 .addMapping("type1",
                 XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("str_value")
-                        .field("type", "keyword").startObject("fielddata")
-                        .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().startObject("boolean_value")
-                        .field("type", "boolean").endObject().startObject("byte_value").field("type", "byte").startObject("fielddata")
-                        .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().startObject("short_value")
-                        .field("type", "short").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null)
-                        .endObject().endObject().startObject("integer_value").field("type", "integer").startObject("fielddata")
-                        .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().startObject("long_value")
-                        .field("type", "long").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null)
-                        .endObject().endObject().startObject("float_value").field("type", "float").startObject("fielddata")
-                        .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().startObject("double_value")
-                        .field("type", "double").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null)
-                        .endObject().endObject().endObject().endObject().endObject()));
+                        .field("type", "keyword").endObject().startObject("boolean_value").field("type", "boolean").endObject()
+                        .startObject("byte_value").field("type", "byte").endObject().startObject("short_value").field("type", "short")
+                        .endObject().startObject("integer_value").field("type", "integer").endObject().startObject("long_value")
+                        .field("type", "long").endObject().startObject("float_value").field("type", "float").endObject()
+                        .startObject("double_value").field("type", "double").endObject().endObject().endObject().endObject()));
         ensureGreen();
         List<IndexRequestBuilder> builders = new ArrayList<>();
         for (int i = 0; i < 10; i++) {
@@ -715,11 +707,9 @@ public class FieldSortIT extends ESIntegTestCase {
                         .startObject("properties")
                             .startObject("i_value")
                                 .field("type", "integer")
-                                .startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject()
                             .endObject()
                             .startObject("d_value")
                                 .field("type", "float")
-                                .startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject()
                             .endObject()
                         .endObject()
                         .endObject()
@@ -902,17 +892,11 @@ public class FieldSortIT extends ESIntegTestCase {
         assertAcked(prepareCreate("test")
 .addMapping("type1",
                 XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("long_values")
-                        .field("type", "long").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null)
-                        .endObject().endObject().startObject("int_values").field("type", "integer").startObject("fielddata")
-                        .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().startObject("short_values")
-                        .field("type", "short").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null)
-                        .endObject().endObject().startObject("byte_values").field("type", "byte").startObject("fielddata")
-                        .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().startObject("float_values")
-                        .field("type", "float").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null)
-                        .endObject().endObject().startObject("double_values").field("type", "double").startObject("fielddata")
-                        .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().startObject("string_values")
-                        .field("type", "keyword").startObject("fielddata")
-                        .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().endObject().endObject()
+                        .field("type", "long").endObject().startObject("int_values").field("type", "integer").endObject()
+                        .startObject("short_values").field("type", "short").endObject().startObject("byte_values")
+                        .field("type", "byte").endObject().startObject("float_values").field("type", "float").endObject()
+                        .startObject("double_values").field("type", "double").endObject().startObject("string_values")
+                        .field("type", "keyword").endObject().endObject().endObject()
                         .endObject()));
         ensureGreen();
 
@@ -985,7 +969,7 @@ public class FieldSortIT extends ESIntegTestCase {
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
                 .setSize(10)
-                .addSort(SortBuilders.fieldSort("long_values").order(SortOrder.DESC).sortMode("sum"))
+                .addSort(SortBuilders.fieldSort("long_values").order(SortOrder.DESC).sortMode(SortMode.SUM))
                 .execute().actionGet();
 
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L));
@@ -1000,6 +984,42 @@ public class FieldSortIT extends ESIntegTestCase {
         assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3)));
         assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).longValue(), equalTo(2L));
 
+        searchResponse = client().prepareSearch()
+                .setQuery(matchAllQuery())
+                .setSize(10)
+                .addSort(SortBuilders.fieldSort("long_values").order(SortOrder.DESC).sortMode(SortMode.AVG))
+                .execute().actionGet();
+
+        assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L));
+        assertThat(searchResponse.getHits().hits().length, equalTo(3));
+
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
+        assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).longValue(), equalTo(13L));
+
+        assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
+        assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).longValue(), equalTo(6L));
+
+        assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3)));
+        assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).longValue(), equalTo(1L));
+
+        searchResponse = client().prepareSearch()
+                .setQuery(matchAllQuery())
+                .setSize(10)
+                .addSort(SortBuilders.fieldSort("long_values").order(SortOrder.DESC).sortMode(SortMode.MEDIAN))
+                .execute().actionGet();
+
+        assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L));
+        assertThat(searchResponse.getHits().hits().length, equalTo(3));
+
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
+        assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).longValue(), equalTo(13L));
+
+        assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
+        assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).longValue(), equalTo(7L));
+
+        assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3)));
+        assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).longValue(), equalTo(2L));
+
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
                 .setSize(10)
@@ -1220,9 +1240,7 @@ public class FieldSortIT extends ESIntegTestCase {
     public void testSortOnRareField() throws IOException {
         assertAcked(prepareCreate("test").addMapping("type1",
                 XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("string_values")
-                        .field("type", "keyword").startObject("fielddata")
-                        .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().endObject().endObject()
-                        .endObject()));
+                        .field("type", "keyword").endObject().endObject().endObject().endObject()));
         ensureGreen();
         client().prepareIndex("test", "type1", Integer.toString(1)).setSource(jsonBuilder().startObject()
                 .array("string_values", "01", "05", "10", "08")
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java
index d40cbf93002..ed733fd4cd7 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java
@@ -270,7 +270,7 @@ public class GeoDistanceIT extends ESIntegTestCase {
 
         // Order: Asc, Mode: max
         searchResponse = client().prepareSearch("test").setQuery(matchAllQuery())
-                .addSort(SortBuilders.geoDistanceSort("locations", 40.7143528, -74.0059731).order(SortOrder.ASC).sortMode("max"))
+                .addSort(SortBuilders.geoDistanceSort("locations", 40.7143528, -74.0059731).order(SortOrder.ASC).sortMode(SortMode.MAX))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 5);
@@ -296,7 +296,7 @@ public class GeoDistanceIT extends ESIntegTestCase {
 
         // Order: Desc, Mode: min
         searchResponse = client().prepareSearch("test").setQuery(matchAllQuery())
-                .addSort(SortBuilders.geoDistanceSort("locations", 40.7143528, -74.0059731).order(SortOrder.DESC).sortMode("min"))
+                .addSort(SortBuilders.geoDistanceSort("locations", 40.7143528, -74.0059731).order(SortOrder.DESC).sortMode(SortMode.MIN))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 5);
@@ -308,7 +308,7 @@ public class GeoDistanceIT extends ESIntegTestCase {
         assertThat(((Number) searchResponse.getHits().getAt(4).sortValues()[0]).doubleValue(), closeTo(0d, 10d));
 
         searchResponse = client().prepareSearch("test").setQuery(matchAllQuery())
-                .addSort(SortBuilders.geoDistanceSort("locations", 40.7143528, -74.0059731).sortMode("avg").order(SortOrder.ASC))
+                .addSort(SortBuilders.geoDistanceSort("locations", 40.7143528, -74.0059731).sortMode(SortMode.AVG).order(SortOrder.ASC))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 5);
@@ -320,7 +320,7 @@ public class GeoDistanceIT extends ESIntegTestCase {
         assertThat(((Number) searchResponse.getHits().getAt(4).sortValues()[0]).doubleValue(), closeTo(5301d, 10d));
 
         searchResponse = client().prepareSearch("test").setQuery(matchAllQuery())
-                .addSort(SortBuilders.geoDistanceSort("locations", 40.7143528, -74.0059731).sortMode("avg").order(SortOrder.DESC))
+                .addSort(SortBuilders.geoDistanceSort("locations", 40.7143528, -74.0059731).sortMode(SortMode.AVG).order(SortOrder.DESC))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 5);
@@ -333,7 +333,7 @@ public class GeoDistanceIT extends ESIntegTestCase {
 
         try {
                 client().prepareSearch("test").setQuery(matchAllQuery())
-                        .addSort(SortBuilders.geoDistanceSort("locations", 40.7143528, -74.0059731).sortMode("sum"));
+                        .addSort(SortBuilders.geoDistanceSort("locations", 40.7143528, -74.0059731).sortMode(SortMode.SUM));
                 fail("sum should not be supported for sorting by geo distance");
         } catch (IllegalArgumentException e) {
             // expected
@@ -455,7 +455,7 @@ public class GeoDistanceIT extends ESIntegTestCase {
         // Order: Asc, Mode: max
         searchResponse = client()
                 .prepareSearch("companies").setQuery(matchAllQuery()).addSort(SortBuilders.geoDistanceSort("branches.location",
-                        40.7143528, -74.0059731).order(SortOrder.ASC).sortMode("max").setNestedPath("branches"))
+                        40.7143528, -74.0059731).order(SortOrder.ASC).sortMode(SortMode.MAX).setNestedPath("branches"))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 4);
@@ -480,7 +480,7 @@ public class GeoDistanceIT extends ESIntegTestCase {
         // Order: Desc, Mode: min
         searchResponse = client()
                 .prepareSearch("companies").setQuery(matchAllQuery()).addSort(SortBuilders.geoDistanceSort("branches.location",
-                        40.7143528, -74.0059731).order(SortOrder.DESC).sortMode("min").setNestedPath("branches"))
+                        40.7143528, -74.0059731).order(SortOrder.DESC).sortMode(SortMode.MIN).setNestedPath("branches"))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 4);
@@ -492,7 +492,7 @@ public class GeoDistanceIT extends ESIntegTestCase {
 
         searchResponse = client()
                 .prepareSearch("companies").setQuery(matchAllQuery()).addSort(SortBuilders.geoDistanceSort("branches.location",
-                        40.7143528, -74.0059731).sortMode("avg").order(SortOrder.ASC).setNestedPath("branches"))
+                        40.7143528, -74.0059731).sortMode(SortMode.AVG).order(SortOrder.ASC).setNestedPath("branches"))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 4);
@@ -504,7 +504,7 @@ public class GeoDistanceIT extends ESIntegTestCase {
 
         searchResponse = client().prepareSearch("companies")
                 .setQuery(matchAllQuery()).addSort(SortBuilders.geoDistanceSort("branches.location", 40.7143528, -74.0059731)
-                        .setNestedPath("branches").sortMode("avg").order(SortOrder.DESC).setNestedPath("branches"))
+                        .setNestedPath("branches").sortMode(SortMode.AVG).order(SortOrder.DESC).setNestedPath("branches"))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 4);
@@ -517,7 +517,7 @@ public class GeoDistanceIT extends ESIntegTestCase {
         searchResponse = client().prepareSearch("companies").setQuery(matchAllQuery())
                 .addSort(SortBuilders.geoDistanceSort("branches.location", 40.7143528, -74.0059731)
                         .setNestedFilter(termQuery("branches.name", "brooklyn"))
-                        .sortMode("avg").order(SortOrder.ASC).setNestedPath("branches"))
+                        .sortMode(SortMode.AVG).order(SortOrder.ASC).setNestedPath("branches"))
                 .execute().actionGet();
         assertHitCount(searchResponse, 4);
         assertFirstHit(searchResponse, hasId("4"));
@@ -529,7 +529,7 @@ public class GeoDistanceIT extends ESIntegTestCase {
 
         try {
                 client().prepareSearch("companies").setQuery(matchAllQuery())
-                        .addSort(SortBuilders.geoDistanceSort("branches.location", 40.7143528, -74.0059731).sortMode("sum")
+                        .addSort(SortBuilders.geoDistanceSort("branches.location", 40.7143528, -74.0059731).sortMode(SortMode.SUM)
                                 .setNestedPath("branches"));
                 fail("Sum should not be allowed as sort mode");
         } catch (IllegalArgumentException e) {
@@ -567,11 +567,11 @@ public class GeoDistanceIT extends ESIntegTestCase {
         assertHitCount(result, 1);
     }
 
-    private double randomLon() {
+    private static double randomLon() {
         return randomDouble() * 360 - 180;
     }
 
-    private double randomLat() {
+    private static double randomLat() {
         return randomDouble() * 180 - 90;
     }
 
@@ -619,7 +619,7 @@ public class GeoDistanceIT extends ESIntegTestCase {
         }
     }
 
-    private long assertDuelOptimization(SearchResponse resp) {
+    private static long assertDuelOptimization(SearchResponse resp) {
         long matches = -1;
         assertSearchResponse(resp);
         if (matches < 0) {
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
index dc06c43cb85..a88c977cce6 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
@@ -50,6 +50,8 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSort
 import static org.hamcrest.Matchers.closeTo;
 
 public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
+    private static final String LOCATION_FIELD = "location";
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return pluginList(InternalSettingsPlugin.class);
@@ -69,7 +71,7 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
          */
         Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
         Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", "location", "type=geo_point"));
+        assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", LOCATION_FIELD, "type=geo_point"));
         XContentBuilder d1Builder = jsonBuilder();
         GeoPoint[] d1Points = {new GeoPoint(3, 2), new GeoPoint(4, 1)};
         createShuffeldJSONArray(d1Builder, d1Points);
@@ -78,8 +80,8 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
         GeoPoint[] d2Points = {new GeoPoint(5, 1), new GeoPoint(6, 2)};
         createShuffeldJSONArray(d2Builder, d2Points);
 
-        logger.info(d1Builder.string());
-        logger.info(d2Builder.string());
+        logger.info("d1: {}", d1Builder);
+        logger.info("d2: {}", d2Builder);
         indexRandom(true,
                 client().prepareIndex("index", "type", "d1").setSource(d1Builder),
                 client().prepareIndex("index", "type", "d2").setSource(d2Builder));
@@ -95,7 +97,7 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
 
         SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(new GeoDistanceSortBuilder("location", q).sortMode("min").order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MIN).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d1", "d2");
         assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 3, 2, DistanceUnit.KILOMETERS), 0.01d));
@@ -103,7 +105,7 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(new GeoDistanceSortBuilder("location", q).sortMode("min").order(SortOrder.DESC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MIN).order(SortOrder.DESC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d2", "d1");
         assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 1, 5, 1, DistanceUnit.KILOMETERS), 0.01d));
@@ -111,7 +113,7 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(new GeoDistanceSortBuilder("location", q).sortMode("max").order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MAX).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d1", "d2");
         assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 4, 1, DistanceUnit.KILOMETERS), 0.01d));
@@ -119,18 +121,61 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(new GeoDistanceSortBuilder("location", q).sortMode("max").order(SortOrder.DESC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MAX).order(SortOrder.DESC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d2", "d1");
         assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 1, 6, 2, DistanceUnit.KILOMETERS), 0.01d));
         assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 4, 1, DistanceUnit.KILOMETERS), 0.01d));
     }
 
+    public void testSingeToManyAvgMedian() throws ExecutionException, InterruptedException, IOException {
+        /**
+         * q  = (0, 0)
+         *
+         * d1 = (0, 1), (0, 4), (0, 10); so avg. distance is 5, median distance is 4
+         * d2 = (0, 1), (0, 5), (0, 6); so avg. distance is 4, median distance is 5
+         */
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
+        assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", LOCATION_FIELD, "type=geo_point"));
+        XContentBuilder d1Builder = jsonBuilder();
+        GeoPoint[] d1Points = {new GeoPoint(0, 1), new GeoPoint(0, 4), new GeoPoint(0, 10)};
+        createShuffeldJSONArray(d1Builder, d1Points);
+
+        XContentBuilder d2Builder = jsonBuilder();
+        GeoPoint[] d2Points = {new GeoPoint(0, 1), new GeoPoint(0, 5), new GeoPoint(0, 6)};
+        createShuffeldJSONArray(d2Builder, d2Points);
+
+        logger.info("d1: {}", d1Builder);
+        logger.info("d2: {}", d2Builder);
+        indexRandom(true,
+                client().prepareIndex("index", "type", "d1").setSource(d1Builder),
+                client().prepareIndex("index", "type", "d2").setSource(d2Builder));
+        ensureYellow();
+        GeoPoint q = new GeoPoint(0,0);
+
+        SearchResponse searchResponse = client().prepareSearch()
+                .setQuery(matchAllQuery())
+                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.AVG).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .execute().actionGet();
+        assertOrderedSearchHits(searchResponse, "d2", "d1");
+        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(0, 0, 0, 4, DistanceUnit.KILOMETERS), 0.01d));
+        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(0, 0, 0, 5, DistanceUnit.KILOMETERS), 0.01d));
+
+        searchResponse = client().prepareSearch()
+                .setQuery(matchAllQuery())
+                .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MEDIAN).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .execute().actionGet();
+        assertOrderedSearchHits(searchResponse, "d1", "d2");
+        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(0, 0, 0, 4, DistanceUnit.KILOMETERS), 0.01d));
+        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(0, 0, 0, 5, DistanceUnit.KILOMETERS), 0.01d));
+    }
+
     protected void createShuffeldJSONArray(XContentBuilder builder, GeoPoint[] pointsArray) throws IOException {
         List<GeoPoint> points = new ArrayList<>();
         points.addAll(Arrays.asList(pointsArray));
         builder.startObject();
-        builder.startArray("location");
+        builder.startArray(LOCATION_FIELD);
         int numPoints = points.size();
         for (int i = 0; i < numPoints; i++) {
             builder.value(points.remove(randomInt(points.size() - 1)));
@@ -154,7 +199,7 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
          */
         Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
         Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", "location", "type=geo_point"));
+        assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", LOCATION_FIELD, "type=geo_point"));
         XContentBuilder d1Builder = jsonBuilder();
         GeoPoint[] d1Points = {new GeoPoint(2.5, 1), new GeoPoint(2.75, 2), new GeoPoint(3, 3), new GeoPoint(3.25, 4)};
         createShuffeldJSONArray(d1Builder, d1Points);
@@ -177,13 +222,13 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
             int at = randomInt(3 - i);
             if (randomBoolean()) {
                 if (geoDistanceSortBuilder == null) {
-                    geoDistanceSortBuilder = new GeoDistanceSortBuilder("location", qHashes.get(at));
+                    geoDistanceSortBuilder = new GeoDistanceSortBuilder(LOCATION_FIELD, qHashes.get(at));
                 } else {
                     geoDistanceSortBuilder.geohashes(qHashes.get(at));
                 }
             } else {
                 if (geoDistanceSortBuilder == null) {
-                    geoDistanceSortBuilder = new GeoDistanceSortBuilder("location", qPoints.get(at));
+                    geoDistanceSortBuilder = new GeoDistanceSortBuilder(LOCATION_FIELD, qPoints.get(at));
                 } else {
                     geoDistanceSortBuilder.points(qPoints.get(at));
                 }
@@ -194,7 +239,7 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
 
         SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(geoDistanceSortBuilder.sortMode("min").order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(geoDistanceSortBuilder.sortMode(SortMode.MIN).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d1", "d2");
         assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2.5, 1, 2, 1, DistanceUnit.KILOMETERS), 1.e-4));
@@ -202,7 +247,7 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(geoDistanceSortBuilder.sortMode("max").order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(geoDistanceSortBuilder.sortMode(SortMode.MAX).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d1", "d2");
         assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(3.25, 4, 2, 1, DistanceUnit.KILOMETERS), 1.e-4));
@@ -211,61 +256,69 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
     }
 
     public void testSinglePointGeoDistanceSort() throws ExecutionException, InterruptedException, IOException {
-        assertAcked(prepareCreate("index").addMapping("type", "location", "type=geo_point"));
+        assertAcked(prepareCreate("index").addMapping("type", LOCATION_FIELD, "type=geo_point"));
         indexRandom(true,
-                client().prepareIndex("index", "type", "d1").setSource(jsonBuilder().startObject().startObject("location").field("lat", 1).field("lon", 1).endObject().endObject()),
-                client().prepareIndex("index", "type", "d2").setSource(jsonBuilder().startObject().startObject("location").field("lat", 1).field("lon", 2).endObject().endObject()));
+                client().prepareIndex("index", "type", "d1").setSource(jsonBuilder().startObject().startObject(LOCATION_FIELD).field("lat", 1).field("lon", 1).endObject().endObject()),
+                client().prepareIndex("index", "type", "d2").setSource(jsonBuilder().startObject().startObject(LOCATION_FIELD).field("lat", 1).field("lon", 2).endObject().endObject()));
         ensureYellow();
 
         String hashPoint = "s037ms06g7h0";
 
-        GeoDistanceSortBuilder geoDistanceSortBuilder = new GeoDistanceSortBuilder("location", hashPoint);
+        GeoDistanceSortBuilder geoDistanceSortBuilder = new GeoDistanceSortBuilder(LOCATION_FIELD, hashPoint);
 
         SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(geoDistanceSortBuilder.sortMode("min").order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(geoDistanceSortBuilder.sortMode(SortMode.MIN).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
                 .execute().actionGet();
         checkCorrectSortOrderForGeoSort(searchResponse);
 
-        geoDistanceSortBuilder = new GeoDistanceSortBuilder("location", new GeoPoint(2, 2));
+        geoDistanceSortBuilder = new GeoDistanceSortBuilder(LOCATION_FIELD, new GeoPoint(2, 2));
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(geoDistanceSortBuilder.sortMode("min").order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(geoDistanceSortBuilder.sortMode(SortMode.MIN).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
                 .execute().actionGet();
         checkCorrectSortOrderForGeoSort(searchResponse);
 
-        geoDistanceSortBuilder = new GeoDistanceSortBuilder("location", 2, 2);
+        geoDistanceSortBuilder = new GeoDistanceSortBuilder(LOCATION_FIELD, 2, 2);
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(geoDistanceSortBuilder.sortMode("min").order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
+                .addSort(geoDistanceSortBuilder.sortMode(SortMode.MIN).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS))
                 .execute().actionGet();
         checkCorrectSortOrderForGeoSort(searchResponse);
 
         searchResponse = client()
                 .prepareSearch()
                 .setSource(
-                        new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort("location", 2.0, 2.0)
+                        new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort(LOCATION_FIELD, 2.0, 2.0)
                                 .unit(DistanceUnit.KILOMETERS).geoDistance(GeoDistance.PLANE))).execute().actionGet();
         checkCorrectSortOrderForGeoSort(searchResponse);
 
         searchResponse = client()
                 .prepareSearch()
                 .setSource(
-                        new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort("location", "s037ms06g7h0")
+                        new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort(LOCATION_FIELD, "s037ms06g7h0")
                                 .unit(DistanceUnit.KILOMETERS).geoDistance(GeoDistance.PLANE))).execute().actionGet();
         checkCorrectSortOrderForGeoSort(searchResponse);
 
         searchResponse = client()
                 .prepareSearch()
                 .setSource(
-                        new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort("location", 2.0, 2.0)
+                        new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort(LOCATION_FIELD, 2.0, 2.0)
                                 .unit(DistanceUnit.KILOMETERS).geoDistance(GeoDistance.PLANE))).execute().actionGet();
         checkCorrectSortOrderForGeoSort(searchResponse);
+
+        searchResponse = client()
+                .prepareSearch()
+                .setSource(
+                        new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort(LOCATION_FIELD, 2.0, 2.0)
+                                .unit(DistanceUnit.KILOMETERS).geoDistance(GeoDistance.PLANE)
+                                .ignoreMalformed(true).coerce(true))).execute().actionGet();
+        checkCorrectSortOrderForGeoSort(searchResponse);
     }
 
-    private void checkCorrectSortOrderForGeoSort(SearchResponse searchResponse) {
+    private static void checkCorrectSortOrderForGeoSort(SearchResponse searchResponse) {
         assertOrderedSearchHits(searchResponse, "d2", "d1");
         assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 1, 2, DistanceUnit.KILOMETERS), 1.e-4));
         assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 1, 1, DistanceUnit.KILOMETERS), 1.e-4));
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
index e957db58b38..d2a7624e08b 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
@@ -20,14 +20,16 @@
 package org.elasticsearch.search.sort;
 
 
+import org.apache.lucene.search.SortField;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.geo.RandomGeoGenerator;
 
 import java.io.IOException;
@@ -37,6 +39,10 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
 
     @Override
     protected GeoDistanceSortBuilder createTestItem() {
+        return randomGeoDistanceSortBuilder();
+    }
+
+    public static GeoDistanceSortBuilder randomGeoDistanceSortBuilder() {
         String fieldName = randomAsciiOfLengthBetween(1, 10);
         GeoDistanceSortBuilder result = null;
 
@@ -60,7 +66,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
                 break;
             default:
                 throw new IllegalStateException("one of three geo initialisation strategies must be used");
-           
+
         }
         if (randomBoolean()) {
             result.geoDistance(geoDistance(result.geoDistance()));
@@ -69,7 +75,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.unit(unit(result.unit()));
         }
         if (randomBoolean()) {
-            result.order(RandomSortDataGenerator.order(result.order()));
+            result.order(RandomSortDataGenerator.order(null));
         }
         if (randomBoolean()) {
             result.sortMode(mode(result.sortMode()));
@@ -86,20 +92,26 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
         if (randomBoolean()) {
             result.ignoreMalformed(! result.ignoreMalformed());
         }
-        
+
         return result;
     }
 
-    private static String mode(String original) {
-        String[] modes = {"MIN", "MAX", "AVG"};
-        String mode = ESTestCase.randomFrom(modes);
-        while (mode.equals(original)) {
-            mode = ESTestCase.randomFrom(modes);
-        }
-        return mode;
+    @Override
+    protected MappedFieldType provideMappedFieldType(String name) {
+        MappedFieldType clone = GeoPointFieldMapper.Defaults.FIELD_TYPE.clone();
+        clone.setName(name);
+        return clone;
     }
 
-    private DistanceUnit unit(DistanceUnit original) {
+    private static SortMode mode(SortMode original) {
+        SortMode result;
+        do {
+            result = randomFrom(SortMode.values());
+        } while (result == SortMode.SUM || result == original);
+        return result;
+    }
+
+    private static DistanceUnit unit(DistanceUnit original) {
         int id = -1;
         while (id == -1 || (original != null && original.ordinal() == id)) {
             id = randomIntBetween(0, DistanceUnit.values().length - 1);
@@ -107,7 +119,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
         return DistanceUnit.values()[id];
     }
 
-    private GeoPoint[] points(GeoPoint[] original) {
+    private static GeoPoint[] points(GeoPoint[] original) {
         GeoPoint[] result = null;
         while (result == null || Arrays.deepEquals(original, result)) {
             int count = randomIntBetween(1, 10);
@@ -119,7 +131,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
         return result;
     }
 
-    private GeoDistance geoDistance(GeoDistance original) {
+    private static GeoDistance geoDistance(GeoDistance original) {
         int id = -1;
         while (id == -1 || (original != null && original.ordinal() == id)) {
             id = randomIntBetween(0, GeoDistance.values().length - 1);
@@ -169,13 +181,13 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             break;
         }
         return result;
-
     }
 
-    @SuppressWarnings("unchecked")
     @Override
-    protected Class<GeoDistanceSortBuilder> getPrototype() {
-        return (Class<GeoDistanceSortBuilder>) GeoDistanceSortBuilder.PROTOTYPE.getClass();
+    protected void sortFieldAssertions(GeoDistanceSortBuilder builder, SortField sortField) throws IOException {
+        assertEquals(SortField.Type.CUSTOM, sortField.getType());
+        assertEquals(builder.order() == SortOrder.ASC ? false : true, sortField.getReverse());
+        assertEquals(builder.fieldName(), sortField.getField());
     }
 
     public void testSortModeSumIsRejectedInSetter() {
@@ -183,70 +195,71 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
         GeoPoint point = RandomGeoGenerator.randomPoint(getRandom());
         builder.point(point.getLat(), point.getLon());
         try {
-            builder.sortMode("SUM");
+            builder.sortMode(SortMode.SUM);
             fail("sort mode sum should not be supported");
           } catch (IllegalArgumentException e) {
               // all good
           }
     }
-    
+
     public void testSortModeSumIsRejectedInJSON() throws IOException {
-        String json = "{\n" + 
-                "  \"testname\" : [ {\n" + 
-                "    \"lat\" : -6.046997540714173,\n" + 
-                "    \"lon\" : -51.94128329747579\n" + 
-                "  } ],\n" + 
-                "  \"unit\" : \"m\",\n" + 
-                "  \"distance_type\" : \"sloppy_arc\",\n" + 
-                "  \"reverse\" : true,\n" + 
-                "  \"mode\" : \"SUM\",\n" + 
-                "  \"coerce\" : false,\n" + 
-                "  \"ignore_malformed\" : false\n" + 
+        String json = "{\n" +
+                "  \"testname\" : [ {\n" +
+                "    \"lat\" : -6.046997540714173,\n" +
+                "    \"lon\" : -51.94128329747579\n" +
+                "  } ],\n" +
+                "  \"unit\" : \"m\",\n" +
+                "  \"distance_type\" : \"sloppy_arc\",\n" +
+                "  \"reverse\" : true,\n" +
+                "  \"mode\" : \"SUM\",\n" +
+                "  \"coerce\" : false,\n" +
+                "  \"ignore_malformed\" : false\n" +
                 "}";
         XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
         itemParser.nextToken();
-        
+
         QueryParseContext context = new QueryParseContext(indicesQueriesRegistry);
         context.reset(itemParser);
 
-        try {
-          GeoDistanceSortBuilder.PROTOTYPE.fromXContent(context, "");
-          fail("sort mode sum should not be supported");
-        } catch (IllegalArgumentException e) {
-            // all good
-        }
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> GeoDistanceSortBuilder.fromXContent(context, ""));
+        assertEquals("sort_mode [sum] isn't supported for sorting by geo distance", e.getMessage());
     }
-    
+
     public void testGeoDistanceSortCanBeParsedFromGeoHash() throws IOException {
-        String json = "{\n" + 
+        String json = "{\n" +
                 "    \"VDcvDuFjE\" : [ \"7umzzv8eychg\", \"dmdgmt5z13uw\", " +
-                "    \"ezu09wxw6v4c\", \"kc7s3515p6k6\", \"jgeuvjwrmfzn\", \"kcpcfj7ruyf8\" ],\n" + 
-                "    \"unit\" : \"m\",\n" + 
-                "    \"distance_type\" : \"sloppy_arc\",\n" + 
-                "    \"reverse\" : true,\n" + 
-                "    \"mode\" : \"MAX\",\n" + 
-                "    \"nested_filter\" : {\n" + 
-                "      \"ids\" : {\n" + 
-                "        \"type\" : [ ],\n" + 
-                "        \"values\" : [ ],\n" + 
-                "        \"boost\" : 5.711116\n" + 
-                "      }\n" + 
-                "    },\n" + 
-                "    \"coerce\" : false,\n" + 
-                "    \"ignore_malformed\" : true\n" + 
+                "    \"ezu09wxw6v4c\", \"kc7s3515p6k6\", \"jgeuvjwrmfzn\", \"kcpcfj7ruyf8\" ],\n" +
+                "    \"unit\" : \"m\",\n" +
+                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"reverse\" : true,\n" +
+                "    \"mode\" : \"MAX\",\n" +
+                "    \"nested_filter\" : {\n" +
+                "      \"ids\" : {\n" +
+                "        \"type\" : [ ],\n" +
+                "        \"values\" : [ ],\n" +
+                "        \"boost\" : 5.711116\n" +
+                "      }\n" +
+                "    },\n" +
+                "    \"coerce\" : false,\n" +
+                "    \"ignore_malformed\" : true\n" +
                 "  }";
         XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
         itemParser.nextToken();
-        
+
         QueryParseContext context = new QueryParseContext(indicesQueriesRegistry);
         context.reset(itemParser);
 
-        GeoDistanceSortBuilder result = GeoDistanceSortBuilder.PROTOTYPE.fromXContent(context, json);
+        GeoDistanceSortBuilder result = GeoDistanceSortBuilder.fromXContent(context, json);
         assertEquals("[-19.700583312660456, -2.8225036337971687, "
                 + "31.537466906011105, -74.63590376079082, "
                 + "43.71844606474042, -5.548660643398762, "
                 + "-37.20467280596495, 38.71751043945551, "
                 + "-69.44606635719538, 84.25200328230858, "
-                + "-39.03717711567879, 44.74099852144718]", Arrays.toString(result.points()));                
+                + "-39.03717711567879, 44.74099852144718]", Arrays.toString(result.points()));
+    }
+
+    @Override
+    protected GeoDistanceSortBuilder fromXContent(QueryParseContext context, String fieldName) throws IOException {
+        return GeoDistanceSortBuilder.fromXContent(context, fieldName);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java b/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java
index fcd5284119c..a31de756bc9 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java
@@ -19,13 +19,16 @@
 
 package org.elasticsearch.search.sort;
 
-import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.query.IdsQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.HashSet;
+import java.util.Set;
+
 public class RandomSortDataGenerator {
     private RandomSortDataGenerator() {
         // this is a helper class only, doesn't need a constructor
@@ -44,7 +47,7 @@ public class RandomSortDataGenerator {
                 break;
             default:
             case 2:
-                nested = new TermQueryBuilder(ESTestCase.randomAsciiOfLengthBetween(1, 10), ESTestCase.randomAsciiOfLengthBetween(1, 10));
+                nested = new TermQueryBuilder(ESTestCase.randomAsciiOfLengthBetween(1, 10), ESTestCase.randomDouble());
                 break;
             }
             nested.boost((float) ESTestCase.randomDoubleBetween(0, 10, false));
@@ -60,26 +63,26 @@ public class RandomSortDataGenerator {
         return nestedPath;
     }
 
-    public static String mode(String original) {
-        String[] modes = {"min", "max", "avg", "sum"};
-        String mode = ESTestCase.randomFrom(modes);
-        while (mode.equals(original)) {
-            mode = ESTestCase.randomFrom(modes);
+    public static SortMode mode(SortMode original) {
+        Set<SortMode> set = new HashSet<>();
+        set.add(original);
+        return mode(set);
+    }
+
+    public static SortMode mode(Set<SortMode> except) {
+        SortMode mode = ESTestCase.randomFrom(SortMode.values());
+        while (except.contains(mode)) {
+            mode = ESTestCase.randomFrom(SortMode.values());
         }
         return mode;
     }
-    
+
     public static Object missing(Object original) {
         Object missing = null;
-        Object otherMissing = null;
-        if (original instanceof BytesRef) {
-            otherMissing = ((BytesRef) original).utf8ToString();
-        } else {
-            otherMissing = original;
-        }
+        Object otherMissing = original;
 
         while (missing == null || missing.equals(otherMissing)) {
-          int missingId = ESTestCase.randomIntBetween(0, 3);
+          int missingId = ESTestCase.randomIntBetween(0, 4);
           switch (missingId) {
           case 0:
               missing = ("_last");
@@ -91,19 +94,27 @@ public class RandomSortDataGenerator {
               missing = ESTestCase.randomAsciiOfLength(10);
               break;
           case 3:
+              missing = ESTestCase.randomUnicodeOfCodepointLengthBetween(5, 15);
+              break;
+          case 4:
               missing = ESTestCase.randomInt();
               break;
           default:
               throw new IllegalStateException("Unknown missing type.");
-              
+
           }
         }
         return missing;
     }
-    
-    public static SortOrder order(SortOrder original) {
-        SortOrder order = SortOrder.ASC;
-        if (order.equals(original)) {
+
+    /**
+     * return a random {@link SortOrder} settings, except the one provided by parameter if set
+     */
+    public static SortOrder order(@Nullable SortOrder original) {
+        if (original == null) {
+            return ESTestCase.randomBoolean() ? SortOrder.ASC : SortOrder.DESC;
+        }
+        if (original.equals(SortOrder.ASC)) {
             return SortOrder.DESC;
         } else {
             return SortOrder.ASC;
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
new file mode 100644
index 00000000000..7b81b310901
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.sort;
+
+
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.junit.Rule;
+import org.junit.rules.ExpectedException;
+
+import java.io.IOException;
+
+public class ScoreSortBuilderTests extends AbstractSortTestCase<ScoreSortBuilder> {
+
+    @Override
+    protected ScoreSortBuilder createTestItem() {
+        return randomScoreSortBuilder();
+    }
+
+    public static ScoreSortBuilder randomScoreSortBuilder() {
+        return new ScoreSortBuilder().order(randomBoolean() ? SortOrder.ASC : SortOrder.DESC);
+    }
+
+    @Override
+    protected ScoreSortBuilder mutate(ScoreSortBuilder original) throws IOException {
+        ScoreSortBuilder result = new ScoreSortBuilder();
+        result.order(RandomSortDataGenerator.order(original.order()));
+        return result;
+    }
+
+    @Rule
+    public ExpectedException exceptionRule = ExpectedException.none();
+
+    /**
+     * test passing null to {@link ScoreSortBuilder#order(SortOrder)} is illegal
+     */
+    public void testIllegalOrder() {
+            exceptionRule.expect(NullPointerException.class);
+            exceptionRule.expectMessage("sort order cannot be null.");
+            new ScoreSortBuilder().order(null);
+    }
+
+    /**
+     * test parsing order parameter if specified as `order` field in the json
+     * instead of the `reverse` field that we render in toXContent
+     */
+    public void testParseOrder() throws IOException {
+        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry);
+        context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY));
+        SortOrder order = randomBoolean() ? SortOrder.ASC : SortOrder.DESC;
+        String scoreSortString = "{ \"_score\": { \"order\": \""+ order.toString() +"\" }}";
+        XContentParser parser = XContentFactory.xContent(scoreSortString).createParser(scoreSortString);
+        // need to skip until parser is located on second START_OBJECT
+        parser.nextToken();
+        parser.nextToken();
+        parser.nextToken();
+
+        context.reset(parser);
+        ScoreSortBuilder scoreSort = ScoreSortBuilder.fromXContent(context, "_score");
+        assertEquals(order, scoreSort.order());
+    }
+
+    @Override
+    protected void sortFieldAssertions(ScoreSortBuilder builder, SortField sortField) {
+        assertEquals(SortField.Type.SCORE, sortField.getType());
+        assertEquals(builder.order() == SortOrder.DESC ? false : true, sortField.getReverse());
+    }
+
+    @Override
+    protected ScoreSortBuilder fromXContent(QueryParseContext context, String fieldName) throws IOException {
+        return ScoreSortBuilder.fromXContent(context, fieldName);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
new file mode 100644
index 00000000000..0e04dc9e4da
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.sort;
+
+
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
+import org.junit.Rule;
+import org.junit.rules.ExpectedException;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuilder> {
+
+    @Override
+    protected ScriptSortBuilder createTestItem() {
+        return randomScriptSortBuilder();
+    }
+
+    public static ScriptSortBuilder randomScriptSortBuilder() {
+        ScriptSortType type = randomBoolean() ? ScriptSortType.NUMBER : ScriptSortType.STRING;
+        ScriptSortBuilder builder = new ScriptSortBuilder(new Script(randomAsciiOfLengthBetween(5, 10)),
+                type);
+        if (randomBoolean()) {
+                builder.order(RandomSortDataGenerator.order(null));
+        }
+        if (randomBoolean()) {
+            if (type == ScriptSortType.NUMBER) {
+                builder.sortMode(RandomSortDataGenerator.mode(builder.sortMode()));
+            } else {
+                Set<SortMode> exceptThis = new HashSet<>();
+                exceptThis.add(SortMode.SUM);
+                exceptThis.add(SortMode.AVG);
+                exceptThis.add(SortMode.MEDIAN);
+                builder.sortMode(RandomSortDataGenerator.mode(exceptThis));
+            }
+        }
+        if (randomBoolean()) {
+            builder.setNestedFilter(RandomSortDataGenerator.nestedFilter(builder.getNestedFilter()));
+        }
+        if (randomBoolean()) {
+            builder.setNestedPath(RandomSortDataGenerator.randomAscii(builder.getNestedPath()));
+        }
+        return builder;
+    }
+
+    @Override
+    protected ScriptSortBuilder mutate(ScriptSortBuilder original) throws IOException {
+        ScriptSortBuilder result;
+        if (randomBoolean()) {
+            // change one of the constructor args, copy the rest over
+            Script script = original.script();
+            ScriptSortType type = original.type();
+            if (randomBoolean()) {
+                result = new ScriptSortBuilder(new Script(script.getScript() + "_suffix"), type);
+            } else {
+                result = new ScriptSortBuilder(script, type.equals(ScriptSortType.NUMBER) ? ScriptSortType.STRING : ScriptSortType.NUMBER);
+            }
+            result.order(original.order());
+            if (original.sortMode() != null && result.type() == ScriptSortType.NUMBER) {
+                result.sortMode(original.sortMode());
+            }
+            result.setNestedFilter(original.getNestedFilter());
+            result.setNestedPath(original.getNestedPath());
+            return result;
+        }
+        result = new ScriptSortBuilder(original);
+        switch (randomIntBetween(0, 3)) {
+            case 0:
+                if (original.order() == SortOrder.ASC) {
+                    result.order(SortOrder.DESC);
+                } else {
+                    result.order(SortOrder.ASC);
+                }
+                break;
+            case 1:
+                if (original.type() == ScriptSortType.NUMBER) {
+                    result.sortMode(RandomSortDataGenerator.mode(original.sortMode()));
+                } else {
+                    // script sort type String only allows MIN and MAX, so we only switch
+                    if (original.sortMode() == SortMode.MIN) {
+                        result.sortMode(SortMode.MAX);
+                    } else {
+                        result.sortMode(SortMode.MIN);
+                    }
+                }
+                break;
+            case 2:
+                result.setNestedFilter(RandomSortDataGenerator.nestedFilter(original.getNestedFilter()));
+                break;
+            case 3:
+                result.setNestedPath(original.getNestedPath() + "_some_suffix");
+                break;
+        }
+        return result;
+    }
+
+    @Override
+    protected void sortFieldAssertions(ScriptSortBuilder builder, SortField sortField) throws IOException {
+        assertEquals(SortField.Type.CUSTOM, sortField.getType());
+        assertEquals(builder.order() == SortOrder.ASC ? false : true, sortField.getReverse());
+    }
+
+    public void testScriptSortType() {
+        // we rely on these ordinals in serialization, so changing them breaks bwc.
+        assertEquals(0, ScriptSortType.STRING.ordinal());
+        assertEquals(1, ScriptSortType.NUMBER.ordinal());
+
+        assertEquals("string", ScriptSortType.STRING.toString());
+        assertEquals("number", ScriptSortType.NUMBER.toString());
+
+        assertEquals(ScriptSortType.STRING, ScriptSortType.fromString("string"));
+        assertEquals(ScriptSortType.STRING, ScriptSortType.fromString("String"));
+        assertEquals(ScriptSortType.STRING, ScriptSortType.fromString("STRING"));
+        assertEquals(ScriptSortType.NUMBER, ScriptSortType.fromString("number"));
+        assertEquals(ScriptSortType.NUMBER, ScriptSortType.fromString("Number"));
+        assertEquals(ScriptSortType.NUMBER, ScriptSortType.fromString("NUMBER"));
+    }
+
+    @Rule
+    public ExpectedException exceptionRule = ExpectedException.none();
+
+    public void testScriptSortTypeNull() {
+        exceptionRule.expect(NullPointerException.class);
+        exceptionRule.expectMessage("input string is null");
+        ScriptSortType.fromString(null);
+    }
+
+    public void testScriptSortTypeIllegalArgument() {
+        exceptionRule.expect(IllegalArgumentException.class);
+        exceptionRule.expectMessage("Unknown ScriptSortType [xyz]");
+        ScriptSortType.fromString("xyz");
+    }
+
+    public void testParseJson() throws IOException {
+        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry);
+        context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY));
+        String scriptSort = "{\n" +
+                "\"_script\" : {\n" +
+                    "\"type\" : \"number\",\n" +
+                    "\"script\" : {\n" +
+                        "\"inline\": \"doc['field_name'].value * factor\",\n" +
+                        "\"params\" : {\n" +
+                            "\"factor\" : 1.1\n" +
+                            "}\n" +
+                    "},\n" +
+                    "\"mode\" : \"max\",\n" +
+                    "\"order\" : \"asc\"\n" +
+                "} }\n";
+        XContentParser parser = XContentFactory.xContent(scriptSort).createParser(scriptSort);
+        parser.nextToken();
+        parser.nextToken();
+        parser.nextToken();
+
+        context.reset(parser);
+        ScriptSortBuilder builder = ScriptSortBuilder.fromXContent(context, null);
+        assertEquals("doc['field_name'].value * factor", builder.script().getScript());
+        assertNull(builder.script().getLang());
+        assertEquals(1.1, builder.script().getParams().get("factor"));
+        assertEquals(ScriptType.INLINE, builder.script().getType());
+        assertEquals(ScriptSortType.NUMBER, builder.type());
+        assertEquals(SortOrder.ASC, builder.order());
+        assertEquals(SortMode.MAX, builder.sortMode());
+        assertNull(builder.getNestedFilter());
+        assertNull(builder.getNestedPath());
+    }
+
+    public void testParseJsonOldStyle() throws IOException {
+        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry);
+        context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY));
+        String scriptSort = "{\n" +
+                "\"_script\" : {\n" +
+                    "\"type\" : \"number\",\n" +
+                    "\"script\" : \"doc['field_name'].value * factor\",\n" +
+                    "\"params\" : {\n" +
+                        "\"factor\" : 1.1\n" +
+                    "},\n" +
+                    "\"mode\" : \"max\",\n" +
+                    "\"order\" : \"asc\"\n" +
+                "} }\n";
+        XContentParser parser = XContentFactory.xContent(scriptSort).createParser(scriptSort);
+        parser.nextToken();
+        parser.nextToken();
+        parser.nextToken();
+
+        context.reset(parser);
+        ScriptSortBuilder builder = ScriptSortBuilder.fromXContent(context, null);
+        assertEquals("doc['field_name'].value * factor", builder.script().getScript());
+        assertNull(builder.script().getLang());
+        assertEquals(1.1, builder.script().getParams().get("factor"));
+        assertEquals(ScriptType.INLINE, builder.script().getType());
+        assertEquals(ScriptSortType.NUMBER, builder.type());
+        assertEquals(SortOrder.ASC, builder.order());
+        assertEquals(SortMode.MAX, builder.sortMode());
+        assertNull(builder.getNestedFilter());
+        assertNull(builder.getNestedPath());
+    }
+
+    public void testParseBadFieldNameExceptions() throws IOException {
+        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry);
+        context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY));
+        String scriptSort = "{\"_script\" : {" + "\"bad_field\" : \"number\"" + "} }";
+        XContentParser parser = XContentFactory.xContent(scriptSort).createParser(scriptSort);
+        parser.nextToken();
+        parser.nextToken();
+        parser.nextToken();
+
+        context.reset(parser);
+        exceptionRule.expect(ParsingException.class);
+        exceptionRule.expectMessage("failed to parse field [bad_field]");
+        ScriptSortBuilder.fromXContent(context, null);
+    }
+
+    public void testParseBadFieldNameExceptionsOnStartObject() throws IOException {
+        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry);
+        context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY));
+
+        String scriptSort = "{\"_script\" : {" + "\"bad_field\" : { \"order\" : \"asc\" } } }";
+        XContentParser parser = XContentFactory.xContent(scriptSort).createParser(scriptSort);
+        parser.nextToken();
+        parser.nextToken();
+        parser.nextToken();
+
+        context.reset(parser);
+        exceptionRule.expect(ParsingException.class);
+        exceptionRule.expectMessage("failed to parse field [bad_field]");
+        ScriptSortBuilder.fromXContent(context, null);
+    }
+
+    public void testParseUnexpectedToken() throws IOException {
+        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry);
+        context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY));
+
+        String scriptSort = "{\"_script\" : {" + "\"script\" : [ \"order\" : \"asc\" ] } }";
+        XContentParser parser = XContentFactory.xContent(scriptSort).createParser(scriptSort);
+        parser.nextToken();
+        parser.nextToken();
+        parser.nextToken();
+
+        context.reset(parser);
+        exceptionRule.expect(ParsingException.class);
+        exceptionRule.expectMessage("unexpected token [START_ARRAY]");
+        ScriptSortBuilder.fromXContent(context, null);
+    }
+
+    /**
+     * script sort of type {@link ScriptSortType} does not work with {@link SortMode#AVG}, {@link SortMode#MEDIAN} or {@link SortMode#SUM}
+     */
+    public void testBadSortMode() throws IOException {
+        ScriptSortBuilder builder = new ScriptSortBuilder(new Script("something"), ScriptSortType.STRING);
+        exceptionRule.expect(IllegalArgumentException.class);
+        exceptionRule.expectMessage("script sort of type [string] doesn't support mode");
+        builder.sortMode(SortMode.fromString(randomFrom(new String[]{"avg", "median", "sum"})));
+    }
+
+    @Override
+    protected ScriptSortBuilder fromXContent(QueryParseContext context, String fieldName) throws IOException {
+        return ScriptSortBuilder.fromXContent(context, fieldName);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java
new file mode 100644
index 00000000000..7d182a5a8b0
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java
@@ -0,0 +1,247 @@
+/*
+x * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.sort;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+public class SortBuilderTests extends ESTestCase {
+
+    private static final int NUMBER_OF_RUNS = 20;
+
+    protected static NamedWriteableRegistry namedWriteableRegistry;
+
+    static IndicesQueriesRegistry indicesQueriesRegistry;
+
+    @BeforeClass
+    public static void init() {
+        namedWriteableRegistry = new NamedWriteableRegistry();
+        indicesQueriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry).buildQueryParserRegistry();
+    }
+
+    @AfterClass
+    public static void afterClass() throws Exception {
+        namedWriteableRegistry = null;
+    }
+
+    /**
+     * test two syntax variations:
+     * - "sort" : "fieldname"
+     * - "sort" : { "fieldname" : "asc" }
+     */
+    public void testSingleFieldSort() throws IOException {
+        SortOrder order = randomBoolean() ? SortOrder.ASC : SortOrder.DESC;
+        String json = "{ \"sort\" : { \"field1\" : \"" + order + "\" }}";
+        List<SortBuilder<?>> result = parseSort(json);
+        assertEquals(1, result.size());
+        SortBuilder<?> sortBuilder = result.get(0);
+        assertEquals(new FieldSortBuilder("field1").order(order), sortBuilder);
+
+        json = "{ \"sort\" : \"field1\" }";
+        result = parseSort(json);
+        assertEquals(1, result.size());
+        sortBuilder = result.get(0);
+        assertEquals(new FieldSortBuilder("field1"), sortBuilder);
+
+        // one element array, see https://github.com/elastic/elasticsearch/issues/17257
+        json = "{ \"sort\" : [\"field1\"] }";
+        result = parseSort(json);
+        assertEquals(1, result.size());
+        sortBuilder = result.get(0);
+        assertEquals(new FieldSortBuilder("field1"), sortBuilder);
+
+        json = "{ \"sort\" : { \"_doc\" : \"" + order + "\" }}";
+        result = parseSort(json);
+        assertEquals(1, result.size());
+        sortBuilder = result.get(0);
+        assertEquals(new FieldSortBuilder("_doc").order(order), sortBuilder);
+
+        json = "{ \"sort\" : \"_doc\" }";
+        result = parseSort(json);
+        assertEquals(1, result.size());
+        sortBuilder = result.get(0);
+        assertEquals(new FieldSortBuilder("_doc"), sortBuilder);
+
+        json = "{ \"sort\" : { \"_score\" : \"" + order +"\" }}";
+        result = parseSort(json);
+        assertEquals(1, result.size());
+        sortBuilder = result.get(0);
+        assertEquals(new ScoreSortBuilder().order(order), sortBuilder);
+
+        json = "{ \"sort\" : \"_score\" }";
+        result = parseSort(json);
+        assertEquals(1, result.size());
+        sortBuilder = result.get(0);
+        assertEquals(new ScoreSortBuilder(), sortBuilder);
+
+        // test two spellings for _geo_disctance
+        json = "{ \"sort\" : ["
+                + "{\"_geoDistance\" : {"
+                +       "\"pin.location\" : \"40,-70\" } }"
+          + "] }";
+        result = parseSort(json);
+        assertEquals(1, result.size());
+        sortBuilder = result.get(0);
+        assertEquals(new GeoDistanceSortBuilder("pin.location", 40, -70), sortBuilder);
+
+        json = "{ \"sort\" : ["
+                + "{\"_geo_distance\" : {"
+                +       "\"pin.location\" : \"40,-70\" } }"
+          + "] }";
+        result = parseSort(json);
+        assertEquals(1, result.size());
+        sortBuilder = result.get(0);
+        assertEquals(new GeoDistanceSortBuilder("pin.location", 40, -70), sortBuilder);
+    }
+
+    /**
+     * test random syntax variations
+     */
+    public void testRandomSortBuilders() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) {
+            List<SortBuilder<?>> testBuilders = randomSortBuilderList();
+            XContentBuilder xContentBuilder = XContentFactory.jsonBuilder();
+            xContentBuilder.startObject();
+            if (testBuilders.size() > 1) {
+                xContentBuilder.startArray("sort");
+            } else {
+                xContentBuilder.field("sort");
+            }
+            for (SortBuilder<?> builder : testBuilders) {
+                if (builder instanceof ScoreSortBuilder || builder instanceof FieldSortBuilder) {
+                    switch (randomIntBetween(0, 2)) {
+                    case 0:
+                        if (builder instanceof ScoreSortBuilder) {
+                            xContentBuilder.value("_score");
+                        } else {
+                            xContentBuilder.value(((FieldSortBuilder) builder).getFieldName());
+                        }
+                        break;
+                    case 1:
+                        xContentBuilder.startObject();
+                        if (builder instanceof ScoreSortBuilder) {
+                            xContentBuilder.field("_score");
+                        } else {
+                            xContentBuilder.field(((FieldSortBuilder) builder).getFieldName());
+                        }
+                        xContentBuilder.value(builder.order());
+                        xContentBuilder.endObject();
+                        break;
+                    case 2:
+                        builder.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
+                        break;
+                    }
+                } else {
+                    builder.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
+                }
+            }
+            if (testBuilders.size() > 1) {
+                xContentBuilder.endArray();
+            }
+            xContentBuilder.endObject();
+            List<SortBuilder<?>> parsedSort = parseSort(xContentBuilder.string());
+            assertEquals(testBuilders.size(), parsedSort.size());
+            Iterator<SortBuilder<?>> iterator = testBuilders.iterator();
+            for (SortBuilder<?> parsedBuilder : parsedSort) {
+                assertEquals(iterator.next(), parsedBuilder);
+            }
+        }
+    }
+
+    public static List<SortBuilder<?>> randomSortBuilderList() {
+        int size = randomIntBetween(1, 5);
+        List<SortBuilder<?>> list = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            switch (randomIntBetween(0, 3)) {
+            case 0:
+                list.add(new ScoreSortBuilder());
+                break;
+            case 1:
+                String fieldName = rarely() ? FieldSortBuilder.DOC_FIELD_NAME : randomAsciiOfLengthBetween(1, 10);
+                list.add(new FieldSortBuilder(fieldName));
+                break;
+            case 2:
+                list.add(GeoDistanceSortBuilderTests.randomGeoDistanceSortBuilder());
+                break;
+            case 3:
+                list.add(ScriptSortBuilderTests.randomScriptSortBuilder());
+                break;
+            default:
+                throw new IllegalStateException("unexpected randomization in tests");
+            }
+        }
+        return list;
+    }
+
+    /**
+     * test array syntax variations:
+     * - "sort" : [ "fieldname", { "fieldname2" : "asc" }, ...]
+     */
+    public void testMultiFieldSort() throws IOException {
+        String json = "{ \"sort\" : ["
+                          + "{ \"post_date\" : {\"order\" : \"asc\"}},"
+                          + "\"user\","
+                          + "{ \"name\" : \"desc\" },"
+                          + "{ \"age\" : \"desc\" },"
+                          + "{"
+                              + "\"_geo_distance\" : {"
+                              + "\"pin.location\" : \"40,-70\" } },"
+                          + "\"_score\""
+                    + "] }";
+        List<SortBuilder<?>> result = parseSort(json);
+        assertEquals(6, result.size());
+        assertEquals(new FieldSortBuilder("post_date").order(SortOrder.ASC), result.get(0));
+        assertEquals(new FieldSortBuilder("user").order(SortOrder.ASC), result.get(1));
+        assertEquals(new FieldSortBuilder("name").order(SortOrder.DESC), result.get(2));
+        assertEquals(new FieldSortBuilder("age").order(SortOrder.DESC), result.get(3));
+        assertEquals(new GeoDistanceSortBuilder("pin.location", new GeoPoint(40, -70)), result.get(4));
+        assertEquals(new ScoreSortBuilder(), result.get(5));
+    }
+
+    private static List<SortBuilder<?>> parseSort(String jsonString) throws IOException {
+        XContentParser itemParser = XContentHelper.createParser(new BytesArray(jsonString));
+        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry);
+        context.reset(itemParser);
+
+        assertEquals(XContentParser.Token.START_OBJECT, itemParser.nextToken());
+        assertEquals(XContentParser.Token.FIELD_NAME, itemParser.nextToken());
+        assertEquals("sort", itemParser.currentName());
+        itemParser.nextToken();
+        return SortBuilder.fromXContent(context);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortModeTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortModeTests.java
new file mode 100644
index 00000000000..29deb6dd76d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/sort/SortModeTests.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.sort;
+
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Rule;
+import org.junit.rules.ExpectedException;
+
+import java.util.Locale;
+
+public class SortModeTests extends ESTestCase {
+
+    @Rule
+    public ExpectedException exceptionRule = ExpectedException.none();
+
+    public void testSortMode() {
+        // we rely on these ordinals in serialization, so changing them breaks bwc.
+        assertEquals(0, SortMode.MIN.ordinal());
+        assertEquals(1, SortMode.MAX.ordinal());
+        assertEquals(2, SortMode.SUM.ordinal());
+        assertEquals(3, SortMode.AVG.ordinal());
+        assertEquals(4, SortMode.MEDIAN.ordinal());
+
+        assertEquals("min", SortMode.MIN.toString());
+        assertEquals("max", SortMode.MAX.toString());
+        assertEquals("sum", SortMode.SUM.toString());
+        assertEquals("avg", SortMode.AVG.toString());
+        assertEquals("median", SortMode.MEDIAN.toString());
+
+        for (SortMode mode : SortMode.values()) {
+            assertEquals(mode, SortMode.fromString(mode.toString()));
+            assertEquals(mode, SortMode.fromString(mode.toString().toUpperCase(Locale.ROOT)));
+        }
+    }
+
+    public void testParseNull() {
+        exceptionRule.expect(NullPointerException.class);
+        exceptionRule.expectMessage("input string is null");
+        SortMode.fromString(null);
+    }
+
+    public void testIllegalArgument() {
+        exceptionRule.expect(IllegalArgumentException.class);
+        exceptionRule.expectMessage("Unknown SortMode [xyz]");
+        SortMode.fromString("xyz");
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java
index e505ec68e6a..2de48decbd8 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java
@@ -38,7 +38,7 @@ public class SortOrderTests extends ESTestCase {
           try (BytesStreamOutput out = new BytesStreamOutput()) {
               unit.writeTo(out);
               try (StreamInput in = StreamInput.wrap(out.bytes())) {
-                  assertThat("Roundtrip serialisation failed.", SortOrder.readOrderFrom(in), equalTo(unit));
+                  assertThat("Roundtrip serialisation failed.", SortOrder.readFromStream(in), equalTo(unit));
               }
           }
         }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java
index cbd7b5468b2..e2c54119623 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java
@@ -19,12 +19,14 @@
 
 package org.elasticsearch.search.sort;
 
+import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.TestSearchContext;
 
@@ -47,10 +49,7 @@ public class SortParserTests extends ESSingleNodeTestCase {
         sortBuilder.field("unit", "km");
         sortBuilder.field("sort_mode", "max");
         sortBuilder.endObject();
-        XContentParser parser = XContentHelper.createParser(sortBuilder.bytes());
-        parser.nextToken();
-        GeoDistanceSortParser geoParser = new GeoDistanceSortParser();
-        geoParser.parse(parser, context);
+        parse(context, sortBuilder);
 
         sortBuilder = jsonBuilder();
         sortBuilder.startObject();
@@ -136,9 +135,11 @@ public class SortParserTests extends ESSingleNodeTestCase {
     }
 
     protected void parse(TestSearchContext context, XContentBuilder sortBuilder) throws Exception {
+        QueryParseContext parseContext = context.getQueryShardContext().parseContext();
         XContentParser parser = XContentHelper.createParser(sortBuilder.bytes());
+        parser.setParseFieldMatcher(ParseFieldMatcher.STRICT);
+        parseContext.reset(parser);
         parser.nextToken();
-        GeoDistanceSortParser geoParser = new GeoDistanceSortParser();
-        geoParser.parse(parser, context);
+        GeoDistanceSortBuilder.fromXContent(parseContext, null);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/stats/SearchStatsUnitTests.java b/core/src/test/java/org/elasticsearch/search/stats/SearchStatsUnitTests.java
index c423513ac72..15fa7e64e3f 100644
--- a/core/src/test/java/org/elasticsearch/search/stats/SearchStatsUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/search/stats/SearchStatsUnitTests.java
@@ -32,9 +32,9 @@ public class SearchStatsUnitTests extends ESTestCase {
         // let's create two dummy search stats with groups
         Map<String, Stats> groupStats1 = new HashMap<>();
         Map<String, Stats> groupStats2 = new HashMap<>();
-        groupStats2.put("group1", new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1));
-        SearchStats searchStats1 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats1);
-        SearchStats searchStats2 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats2);
+        groupStats2.put("group1", new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1));
+        SearchStats searchStats1 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats1);
+        SearchStats searchStats2 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats2);
 
         // adding these two search stats and checking group stats are correct
         searchStats1.add(searchStats2);
@@ -62,5 +62,8 @@ public class SearchStatsUnitTests extends ESTestCase {
         assertEquals(equalTo, stats.getScrollCount());
         assertEquals(equalTo, stats.getScrollTimeInMillis());
         assertEquals(equalTo, stats.getScrollCurrent());
+        assertEquals(equalTo, stats.getSuggestCount());
+        assertEquals(equalTo, stats.getSuggestTimeInMillis());
+        assertEquals(equalTo, stats.getSuggestCurrent());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
new file mode 100644
index 00000000000..dc2b6081bd6
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
@@ -0,0 +1,248 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest;
+
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.script.ScriptContextRegistry;
+import org.elasticsearch.script.ScriptEngineRegistry;
+import org.elasticsearch.script.ScriptServiceTests.TestEngineService;
+import org.elasticsearch.script.ScriptSettings;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
+import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Collections;
+
+import static org.elasticsearch.common.settings.Settings.settingsBuilder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+
+public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBuilder<SB>> extends ESTestCase {
+
+    private static final int NUMBER_OF_TESTBUILDERS = 20;
+    protected static NamedWriteableRegistry namedWriteableRegistry;
+    protected static IndicesQueriesRegistry queriesRegistry;
+    protected static ParseFieldMatcher parseFieldMatcher;
+    protected static Suggesters suggesters;
+
+    /**
+     * setup for the whole base test class
+     */
+    @BeforeClass
+    public static void init() throws IOException {
+        Path genericConfigFolder = createTempDir();
+        Settings baseSettings = settingsBuilder()
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .put(Environment.PATH_CONF_SETTING.getKey(), genericConfigFolder)
+                .build();
+        Environment environment = new Environment(baseSettings);
+        ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry
+                .ScriptEngineRegistration(TestEngineService.class, TestEngineService.TYPES)));
+        ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
+        suggesters = new Suggesters(Collections.emptyMap());
+
+        namedWriteableRegistry = new NamedWriteableRegistry();
+        namedWriteableRegistry.registerPrototype(SuggestionBuilder.class, TermSuggestionBuilder.PROTOTYPE);
+        namedWriteableRegistry.registerPrototype(SuggestionBuilder.class, PhraseSuggestionBuilder.PROTOTYPE);
+        namedWriteableRegistry.registerPrototype(SuggestionBuilder.class, CompletionSuggestionBuilder.PROTOTYPE);
+        queriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry).buildQueryParserRegistry();
+        parseFieldMatcher = ParseFieldMatcher.STRICT;
+    }
+
+    @AfterClass
+    public static void afterClass() throws Exception {
+        namedWriteableRegistry = null;
+        suggesters = null;
+        queriesRegistry = null;
+    }
+
+    /**
+     * Test serialization and deserialization of the suggestion builder
+     */
+    public void testSerialization() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
+            SB original = randomTestBuilder();
+            SB deserialized = serializedCopy(original);
+            assertEquals(deserialized, original);
+            assertEquals(deserialized.hashCode(), original.hashCode());
+            assertNotSame(deserialized, original);
+        }
+    }
+
+    /**
+     * returns a random suggestion builder, setting the common options randomly
+     */
+    protected SB randomTestBuilder() {
+        SB randomSuggestion = randomSuggestionBuilder();
+        return randomSuggestion;
+    }
+
+    public static void setCommonPropertiesOnRandomBuilder(SuggestionBuilder<?> randomSuggestion) {
+        randomSuggestion.text(randomAsciiOfLengthBetween(2, 20)); // have to set the text because we don't know if the global text was set
+        maybeSet(randomSuggestion::prefix, randomAsciiOfLengthBetween(2, 20));
+        maybeSet(randomSuggestion::regex, randomAsciiOfLengthBetween(2, 20));
+        maybeSet(randomSuggestion::analyzer, randomAsciiOfLengthBetween(2, 20));
+        maybeSet(randomSuggestion::size, randomIntBetween(1, 20));
+        maybeSet(randomSuggestion::shardSize, randomIntBetween(1, 20));
+    }
+
+    /**
+     * create a randomized {@link SuggestBuilder} that is used in further tests
+     */
+    protected abstract SB randomSuggestionBuilder();
+
+    /**
+     * Test equality and hashCode properties
+     */
+    public void testEqualsAndHashcode() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
+            SB firstBuilder = randomTestBuilder();
+            assertFalse("suggestion builder is equal to null", firstBuilder.equals(null));
+            assertFalse("suggestion builder is equal to incompatible type", firstBuilder.equals(""));
+            assertTrue("suggestion builder is not equal to self", firstBuilder.equals(firstBuilder));
+            assertThat("same suggestion builder's hashcode returns different values if called multiple times", firstBuilder.hashCode(),
+                    equalTo(firstBuilder.hashCode()));
+        final SB mutate = mutate(firstBuilder);
+        assertThat("different suggestion builders should not be equal", mutate, not(equalTo(firstBuilder)));
+
+            SB secondBuilder = serializedCopy(firstBuilder);
+            assertTrue("suggestion builder is not equal to self", secondBuilder.equals(secondBuilder));
+            assertTrue("suggestion builder is not equal to its copy", firstBuilder.equals(secondBuilder));
+            assertTrue("equals is not symmetric", secondBuilder.equals(firstBuilder));
+            assertThat("suggestion builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
+                    equalTo(firstBuilder.hashCode()));
+
+            SB thirdBuilder = serializedCopy(secondBuilder);
+            assertTrue("suggestion builder is not equal to self", thirdBuilder.equals(thirdBuilder));
+            assertTrue("suggestion builder is not equal to its copy", secondBuilder.equals(thirdBuilder));
+            assertThat("suggestion builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
+                    equalTo(thirdBuilder.hashCode()));
+            assertTrue("equals is not transitive", firstBuilder.equals(thirdBuilder));
+            assertThat("suggestion builder copy's hashcode is different from original hashcode", firstBuilder.hashCode(),
+                    equalTo(thirdBuilder.hashCode()));
+            assertTrue("equals is not symmetric", thirdBuilder.equals(secondBuilder));
+            assertTrue("equals is not symmetric", thirdBuilder.equals(firstBuilder));
+        }
+    }
+
+    /**
+     * creates random suggestion builder, renders it to xContent and back to new
+     * instance that should be equal to original
+     */
+    public void testFromXContent() throws IOException {
+        QueryParseContext context = new QueryParseContext(null);
+        context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY));
+        for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
+            SB suggestionBuilder = randomTestBuilder();
+            XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+            if (randomBoolean()) {
+                xContentBuilder.prettyPrint();
+            }
+            xContentBuilder.startObject();
+            suggestionBuilder.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
+            xContentBuilder.endObject();
+
+            XContentParser parser = XContentHelper.createParser(xContentBuilder.bytes());
+            context.reset(parser);
+            // we need to skip the start object and the name, those will be parsed by outer SuggestBuilder
+            parser.nextToken();
+
+            SuggestionBuilder<?> secondSuggestionBuilder = SuggestionBuilder.fromXContent(context, suggesters);
+            assertNotSame(suggestionBuilder, secondSuggestionBuilder);
+            assertEquals(suggestionBuilder, secondSuggestionBuilder);
+            assertEquals(suggestionBuilder.hashCode(), secondSuggestionBuilder.hashCode());
+        }
+    }
+
+    private SB mutate(SB firstBuilder) throws IOException {
+        SB mutation = serializedCopy(firstBuilder);
+        assertNotSame(mutation, firstBuilder);
+        // change ither one of the shared SuggestionBuilder parameters, or delegate to the specific tests mutate method
+        if (randomBoolean()) {
+            switch (randomIntBetween(0, 5)) {
+            case 0:
+                mutation.text(randomValueOtherThan(mutation.text(), () -> randomAsciiOfLengthBetween(2, 20)));
+                break;
+            case 1:
+                mutation.prefix(randomValueOtherThan(mutation.prefix(), () -> randomAsciiOfLengthBetween(2, 20)));
+                break;
+            case 2:
+                mutation.regex(randomValueOtherThan(mutation.regex(), () -> randomAsciiOfLengthBetween(2, 20)));
+                break;
+            case 3:
+                mutation.analyzer(randomValueOtherThan(mutation.analyzer(), () -> randomAsciiOfLengthBetween(2, 20)));
+                break;
+            case 4:
+                mutation.size(randomValueOtherThan(mutation.size(), () -> randomIntBetween(1, 20)));
+                break;
+            case 5:
+                mutation.shardSize(randomValueOtherThan(mutation.shardSize(), () -> randomIntBetween(1, 20)));
+                break;
+            }
+        } else {
+            mutateSpecificParameters(firstBuilder);
+        }
+        return mutation;
+    }
+
+    /**
+     * take and input {@link SuggestBuilder} and return another one that is
+     * different in one aspect (to test non-equality)
+     */
+    protected abstract void mutateSpecificParameters(SB firstBuilder) throws IOException;
+
+    @SuppressWarnings("unchecked")
+    protected SB serializedCopy(SB original) throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            output.writeSuggestion(original);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+                return (SB) in.readSuggestion();
+            }
+        }
+    }
+
+    protected static QueryParseContext newParseContext(final String xcontent) throws IOException {
+        final QueryParseContext parseContext = new QueryParseContext(queriesRegistry);
+        parseContext.reset(XContentFactory.xContent(xcontent).createParser(xcontent));
+        parseContext.parseFieldMatcher(parseFieldMatcher);
+        return parseContext;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
index 04369348b6f..152c450f6ec 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
@@ -20,10 +20,10 @@ package org.elasticsearch.search.suggest;
 
 import com.carrotsearch.hppc.ObjectLongHashMap;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
 import org.apache.lucene.analysis.TokenStreamToAutomaton;
 import org.apache.lucene.search.suggest.document.ContextSuggestField;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
-import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.admin.indices.segments.IndexShardSegments;
@@ -32,21 +32,22 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.percolate.PercolateResponse;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
-import org.elasticsearch.action.suggest.SuggestRequest;
-import org.elasticsearch.action.suggest.SuggestResponse;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.percolator.PercolatorService;
+import org.elasticsearch.index.percolator.PercolatorFieldMapper;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.suggest.completion.CompletionStats;
 import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
 import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
-import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder.FuzzyOptionsBuilder;
+import org.elasticsearch.search.suggest.completion.FuzzyOptions;
 import org.elasticsearch.search.suggest.completion.context.CategoryContextMapping;
 import org.elasticsearch.search.suggest.completion.context.ContextMapping;
 import org.elasticsearch.search.suggest.completion.context.GeoContextMapping;
@@ -55,6 +56,7 @@ import org.elasticsearch.test.ESIntegTestCase;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
@@ -103,7 +105,7 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
                     ));
         }
         indexRandom(true, indexRequestBuilders);
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg");
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg");
         assertSuggestions("foo", prefix, "suggestion10", "suggestion9", "suggestion8", "suggestion7", "suggestion6");
     }
 
@@ -124,7 +126,7 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
                     ));
         }
         indexRandom(true, indexRequestBuilders);
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).regex("sugg.*es");
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).regex("sugg.*es");
         assertSuggestions("foo", prefix, "sugg10estion", "sugg9estion", "sugg8estion", "sugg7estion", "sugg6estion");
     }
 
@@ -145,7 +147,7 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
                     ));
         }
         indexRandom(true, indexRequestBuilders);
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg", Fuzziness.ONE);
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg", Fuzziness.ONE);
         assertSuggestions("foo", prefix, "sugxgestion10", "sugxgestion9", "sugxgestion8", "sugxgestion7", "sugxgestion6");
     }
 
@@ -171,13 +173,13 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
         for (int i = 0; i < size; i++) {
             outputs[i] = "suggestion" + (numDocs - i);
         }
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sug").size(size);
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sug").size(size);
         assertSuggestions("foo", prefix, outputs);
 
-        CompletionSuggestionBuilder regex = SuggestBuilders.completionSuggestion("foo").field(FIELD).regex("su[g|s]g").size(size);
+        CompletionSuggestionBuilder regex = SuggestBuilders.completionSuggestion(FIELD).regex("su[g|s]g").size(size);
         assertSuggestions("foo", regex, outputs);
 
-        CompletionSuggestionBuilder fuzzyPrefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg", Fuzziness.ONE).size(size);
+        CompletionSuggestionBuilder fuzzyPrefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg", Fuzziness.ONE).size(size);
         assertSuggestions("foo", fuzzyPrefix, outputs);
     }
 
@@ -196,10 +198,12 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
 
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg").size(numDocs).payload("count");
-        SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(prefix).execute().actionGet();
-        assertNoFailures(suggestResponse);
-        CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo");
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg").
+            size(numDocs).payload(Collections.singletonList("count"));
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix))
+            .execute().actionGet();
+        assertNoFailures(searchResponse);
+        CompletionSuggestion completionSuggestion = searchResponse.getSuggest().getSuggestion("foo");
         CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0);
         assertThat(options.getOptions().size(), equalTo(numDocs));
         for (CompletionSuggestion.Entry.Option option : options) {
@@ -208,32 +212,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
         }
     }
 
-    public void testMalformedRequestPayload() throws Exception {
-        final CompletionMappingBuilder mapping = new CompletionMappingBuilder();
-        createIndexAndMapping(mapping);
-        SuggestRequest request = new SuggestRequest(INDEX);
-        XContentBuilder suggest = jsonBuilder().startObject()
-                .startObject("bad-payload")
-                .field("prefix", "sug")
-                .startObject("completion")
-                .field("field", FIELD)
-                .startArray("payload")
-                .startObject()
-                .field("payload", "field")
-                .endObject()
-                .endArray()
-                .endObject()
-                .endObject().endObject();
-        request.suggest(suggest.bytes());
-        ensureGreen();
-
-        SuggestResponse suggestResponse = client().suggest(request).get();
-        assertThat(suggestResponse.getSuccessfulShards(), equalTo(0));
-        for (ShardOperationFailedException exception : suggestResponse.getShardFailures()) {
-            assertThat(exception.reason(), containsString("ParsingException[[completion] failed to parse field [payload]]; nested: IllegalStateException[Can't get text on a START_OBJECT"));
-        }
-    }
-
     public void testMissingPayloadField() throws Exception {
         final CompletionMappingBuilder mapping = new CompletionMappingBuilder();
         createIndexAndMapping(mapping);
@@ -242,10 +220,12 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
                 client().prepareIndex(INDEX, TYPE, "2").setSource(FIELD, "suggestion")
         );
         indexRandom(true, indexRequestBuilders);
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg").payload("test_field");
-        SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(prefix).execute().actionGet();
-        assertNoFailures(suggestResponse);
-        CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo");
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg")
+            .payload(Collections.singletonList("test_field"));
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix))
+            .execute().actionGet();
+        assertNoFailures(searchResponse);
+        CompletionSuggestion completionSuggestion = searchResponse.getSuggest().getSuggestion("foo");
         CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0);
         assertThat(options.getOptions().size(), equalTo(2));
         for (CompletionSuggestion.Entry.Option option : options.getOptions()) {
@@ -279,10 +259,12 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
         indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "2").setSource(source));
         indexRandom(true, indexRequestBuilders);
 
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg").payload("title", "count");
-        SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(prefix).execute().actionGet();
-        assertNoFailures(suggestResponse);
-        CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo");
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg")
+            .payload(Arrays.asList("title", "count"));
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix))
+            .execute().actionGet();
+        assertNoFailures(searchResponse);
+        CompletionSuggestion completionSuggestion = searchResponse.getSuggest().getSuggestion("foo");
         List<CompletionSuggestion.Entry.Option> options = completionSuggestion.getEntries().get(0).getOptions();
         assertThat(options.size(), equalTo(2));
         assertThat(options.get(0).getText().toString(), equalTo("suggestion"));
@@ -324,15 +306,17 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
 
         int suggestionSize = randomIntBetween(1, numDocs);
         int numRequestedPayloadFields = randomIntBetween(2, numPayloadFields);
-        String[] payloadFields = new String[numRequestedPayloadFields];
+        List<String> payloadFields = new ArrayList<>(numRequestedPayloadFields);
         for (int i = 0; i < numRequestedPayloadFields; i++) {
-            payloadFields[i] = "test_field" + i;
+            payloadFields.add("test_field" + i);
         }
 
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg").size(suggestionSize).payload(payloadFields);
-        SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(prefix).execute().actionGet();
-        assertNoFailures(suggestResponse);
-        CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo");
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg")
+            .size(suggestionSize).payload(payloadFields);
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix))
+            .execute().actionGet();
+        assertNoFailures(searchResponse);
+        CompletionSuggestion completionSuggestion = searchResponse.getSuggest().getSuggestion("foo");
         CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0);
         assertThat(options.getOptions().size(), equalTo(suggestionSize));
         int id = numDocs;
@@ -369,7 +353,7 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
                     .setSource(source).execute().actionGet();
         }
 
-        client().prepareIndex(INDEX, PercolatorService.TYPE_NAME, "4")
+        client().prepareIndex(INDEX, PercolatorFieldMapper.TYPE_NAME, "4")
                 .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
                 .execute().actionGet();
 
@@ -428,12 +412,12 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
 
         refresh();
 
-        SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
-                new CompletionSuggestionBuilder("testSuggestions").field(FIELD).text("test").size(10)
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(
+            new SuggestBuilder().addSuggestion("testSuggestions", new CompletionSuggestionBuilder(FIELD).text("test").size(10))
         ).execute().actionGet();
 
-        assertSuggestions(suggestResponse, "testSuggestions", "testing");
-        Suggest.Suggestion.Entry.Option option = suggestResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0);
+        assertSuggestions(searchResponse, "testSuggestions", "testing");
+        Suggest.Suggestion.Entry.Option option = searchResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0);
         assertThat(option, is(instanceOf(CompletionSuggestion.Entry.Option.class)));
         CompletionSuggestion.Entry.Option prefixOption = (CompletionSuggestion.Entry.Option) option;
 
@@ -629,16 +613,16 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
                 .get();
         assertThat(putMappingResponse.isAcknowledged(), is(true));
 
-        SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
-                SuggestBuilders.completionSuggestion("suggs").field(FIELD + ".suggest").text("f").size(10)
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(
+            new SuggestBuilder().addSuggestion("suggs", SuggestBuilders.completionSuggestion(FIELD + ".suggest").text("f").size(10))
         ).execute().actionGet();
-        assertSuggestions(suggestResponse, "suggs");
+        assertSuggestions(searchResponse, "suggs");
 
         client().prepareIndex(INDEX, TYPE, "1").setRefresh(true).setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get();
         ensureGreen(INDEX);
 
-        SuggestResponse afterReindexingResponse = client().prepareSuggest(INDEX).addSuggestion(
-                SuggestBuilders.completionSuggestion("suggs").field(FIELD + ".suggest").text("f").size(10)
+        SearchResponse afterReindexingResponse = client().prepareSearch(INDEX).suggest(
+            new SuggestBuilder().addSuggestion("suggs", SuggestBuilders.completionSuggestion(FIELD + ".suggest").text("f").size(10))
         ).execute().actionGet();
         assertSuggestions(afterReindexingResponse, "suggs", "Foo Fighters");
     }
@@ -654,15 +638,15 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
 
         refresh();
 
-        SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
-                SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nirv").size(10)
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(
+            new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("Nirv").size(10))
         ).execute().actionGet();
-        assertSuggestions(suggestResponse, false, "foo", "Nirvana");
+        assertSuggestions(searchResponse, false, "foo", "Nirvana");
 
-        suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
-                SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nirw", Fuzziness.ONE).size(10)
+        searchResponse = client().prepareSearch(INDEX).suggest(
+            new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("Nirw", Fuzziness.ONE).size(10))
         ).execute().actionGet();
-        assertSuggestions(suggestResponse, false, "foo", "Nirvana");
+        assertSuggestions(searchResponse, false, "foo", "Nirvana");
     }
 
     public void testThatFuzzySuggesterSupportsEditDistances() throws Exception {
@@ -677,16 +661,16 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
         refresh();
 
         // edit distance 1
-        SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
-                SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Norw", Fuzziness.ONE).size(10)
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(
+            new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("Norw", Fuzziness.ONE).size(10))
         ).execute().actionGet();
-        assertSuggestions(suggestResponse, false, "foo");
+        assertSuggestions(searchResponse, false, "foo");
 
         // edit distance 2
-        suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
-                SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Norw", Fuzziness.TWO).size(10)
+        searchResponse = client().prepareSearch(INDEX).suggest(
+            new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("Norw", Fuzziness.TWO).size(10))
         ).execute().actionGet();
-        assertSuggestions(suggestResponse, false, "foo", "Nirvana");
+        assertSuggestions(searchResponse, false, "foo", "Nirvana");
     }
 
     public void testThatFuzzySuggesterSupportsTranspositions() throws Exception {
@@ -700,15 +684,16 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
 
         refresh();
 
-        SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
-                SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nriv", new FuzzyOptionsBuilder().setTranspositions(false)).size(10)
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(
+            new SuggestBuilder().addSuggestion("foo",
+                SuggestBuilders.completionSuggestion(FIELD).prefix("Nriv", FuzzyOptions.builder().setTranspositions(false).build()).size(10))
         ).execute().actionGet();
-        assertSuggestions(suggestResponse, false, "foo");
+        assertSuggestions(searchResponse, false, "foo");
 
-        suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
-                SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nriv", Fuzziness.ONE).size(10)
+        searchResponse = client().prepareSearch(INDEX).suggest(
+            new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("Nriv", Fuzziness.ONE).size(10))
         ).execute().actionGet();
-        assertSuggestions(suggestResponse, false, "foo", "Nirvana");
+        assertSuggestions(searchResponse, false, "foo", "Nirvana");
     }
 
     public void testThatFuzzySuggesterSupportsMinPrefixLength() throws Exception {
@@ -722,15 +707,17 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
 
         refresh();
 
-        SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
-                SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nriva", new FuzzyOptionsBuilder().setFuzzyMinLength(6)).size(10)
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(
+            new SuggestBuilder().addSuggestion("foo",
+                SuggestBuilders.completionSuggestion(FIELD).prefix("Nriva", FuzzyOptions.builder().setFuzzyMinLength(6).build()).size(10))
         ).execute().actionGet();
-        assertSuggestions(suggestResponse, false, "foo");
+        assertSuggestions(searchResponse, false, "foo");
 
-        suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
-                SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nrivan", new FuzzyOptionsBuilder().setFuzzyMinLength(6)).size(10)
+        searchResponse = client().prepareSearch(INDEX).suggest(
+            new SuggestBuilder().addSuggestion("foo",
+                SuggestBuilders.completionSuggestion(FIELD).prefix("Nrivan", FuzzyOptions.builder().setFuzzyMinLength(6).build()).size(10))
         ).execute().actionGet();
-        assertSuggestions(suggestResponse, false, "foo", "Nirvana");
+        assertSuggestions(searchResponse, false, "foo", "Nirvana");
     }
 
     public void testThatFuzzySuggesterSupportsNonPrefixLength() throws Exception {
@@ -744,15 +731,17 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
 
         refresh();
 
-        SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
-                SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nirw", new FuzzyOptionsBuilder().setFuzzyPrefixLength(4)).size(10)
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(
+            new SuggestBuilder().addSuggestion("foo",
+                SuggestBuilders.completionSuggestion(FIELD).prefix("Nirw", FuzzyOptions.builder().setFuzzyPrefixLength(4).build()).size(10))
         ).execute().actionGet();
-        assertSuggestions(suggestResponse, false, "foo");
+        assertSuggestions(searchResponse, false, "foo");
 
-        suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
-                SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nirvo", new FuzzyOptionsBuilder().setFuzzyPrefixLength(4)).size(10)
+        searchResponse = client().prepareSearch(INDEX).suggest(
+            new SuggestBuilder().addSuggestion("foo",
+                SuggestBuilders.completionSuggestion(FIELD).prefix("Nirvo", FuzzyOptions.builder().setFuzzyPrefixLength(4).build()).size(10))
         ).execute().actionGet();
-        assertSuggestions(suggestResponse, false, "foo", "Nirvana");
+        assertSuggestions(searchResponse, false, "foo", "Nirvana");
     }
 
     public void testThatFuzzySuggesterIsUnicodeAware() throws Exception {
@@ -768,20 +757,20 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
 
         // suggestion with a character, which needs unicode awareness
         org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder completionSuggestionBuilder =
-                SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("öööи", new FuzzyOptionsBuilder().setUnicodeAware(true)).size(10);
+                SuggestBuilders.completionSuggestion(FIELD).prefix("öööи", FuzzyOptions.builder().setUnicodeAware(true).build()).size(10);
 
-        SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(completionSuggestionBuilder).execute().actionGet();
-        assertSuggestions(suggestResponse, false, "foo", "ööööö");
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", completionSuggestionBuilder)).execute().actionGet();
+        assertSuggestions(searchResponse, false, "foo", "ööööö");
 
         // removing unicode awareness leads to no result
-        completionSuggestionBuilder = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("öööи", new FuzzyOptionsBuilder().setUnicodeAware(false)).size(10);
-        suggestResponse = client().prepareSuggest(INDEX).addSuggestion(completionSuggestionBuilder).execute().actionGet();
-        assertSuggestions(suggestResponse, false, "foo");
+        completionSuggestionBuilder = SuggestBuilders.completionSuggestion(FIELD).prefix("öööи", FuzzyOptions.builder().setUnicodeAware(false).build()).size(10);
+        searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", completionSuggestionBuilder)).execute().actionGet();
+        assertSuggestions(searchResponse, false, "foo");
 
         // increasing edit distance instead of unicode awareness works again, as this is only a single character
-        completionSuggestionBuilder = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("öööи", new FuzzyOptionsBuilder().setUnicodeAware(false).setFuzziness(Fuzziness.TWO)).size(10);
-        suggestResponse = client().prepareSuggest(INDEX).addSuggestion(completionSuggestionBuilder).execute().actionGet();
-        assertSuggestions(suggestResponse, false, "foo", "ööööö");
+        completionSuggestionBuilder = SuggestBuilders.completionSuggestion(FIELD).prefix("öööи", FuzzyOptions.builder().setUnicodeAware(false).setFuzziness(Fuzziness.TWO).build()).size(10);
+        searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", completionSuggestionBuilder)).execute().actionGet();
+        assertSuggestions(searchResponse, false, "foo", "ööööö");
     }
 
     public void testThatStatsAreWorking() throws Exception {
@@ -809,8 +798,8 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
         refresh();
         ensureGreen();
         // load the fst index into ram
-        client().prepareSuggest(INDEX).addSuggestion(SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("f")).get();
-        client().prepareSuggest(INDEX).addSuggestion(SuggestBuilders.completionSuggestion("foo").field(otherField).prefix("f")).get();
+        client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("f"))).get();
+        client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(otherField).prefix("f"))).get();
 
         // Get all stats
         IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(INDEX).setIndices(INDEX).setCompletion(true).get();
@@ -906,43 +895,42 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
         }
     }
 
-
-    public void assertSuggestions(String suggestionName, SuggestBuilder.SuggestionBuilder suggestBuilder, String... suggestions) {
-        SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestBuilder
-        ).execute().actionGet();
-        assertSuggestions(suggestResponse, suggestionName, suggestions);
-
+    public void assertSuggestions(String suggestionName, SuggestionBuilder suggestBuilder, String... suggestions) {
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion(suggestionName, suggestBuilder)).execute().actionGet();
+        assertSuggestions(searchResponse, suggestionName, suggestions);
     }
+
     public void assertSuggestions(String suggestion, String... suggestions) {
         String suggestionName = RandomStrings.randomAsciiOfLength(random(), 10);
-        CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text(suggestion).size(10);
+        CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion(FIELD).text(suggestion).size(10);
         assertSuggestions(suggestionName, suggestionBuilder, suggestions);
     }
 
     public void assertSuggestionsNotInOrder(String suggestString, String... suggestions) {
         String suggestionName = RandomStrings.randomAsciiOfLength(random(), 10);
-        SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
-                SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text(suggestString).size(10)
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(
+            new SuggestBuilder().addSuggestion(suggestionName,
+                SuggestBuilders.completionSuggestion(FIELD).text(suggestString).size(10))
         ).execute().actionGet();
 
-        assertSuggestions(suggestResponse, false, suggestionName, suggestions);
+        assertSuggestions(searchResponse, false, suggestionName, suggestions);
     }
 
-    static void assertSuggestions(SuggestResponse suggestResponse, String name, String... suggestions) {
-        assertSuggestions(suggestResponse, true, name, suggestions);
+    static void assertSuggestions(SearchResponse searchResponse, String name, String... suggestions) {
+        assertSuggestions(searchResponse, true, name, suggestions);
     }
 
-    private static void assertSuggestions(SuggestResponse suggestResponse, boolean suggestionOrderStrict, String name, String... suggestions) {
-        assertAllSuccessful(suggestResponse);
+    private static void assertSuggestions(SearchResponse searchResponse, boolean suggestionOrderStrict, String name, String... suggestions) {
+        assertAllSuccessful(searchResponse);
 
         List<String> suggestionNames = new ArrayList<>();
-        for (Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>> suggestion : iterableAsArrayList(suggestResponse.getSuggest())) {
+        for (Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>> suggestion : iterableAsArrayList(searchResponse.getSuggest())) {
             suggestionNames.add(suggestion.getName());
         }
         String expectFieldInResponseMsg = String.format(Locale.ROOT, "Expected suggestion named %s in response, got %s", name, suggestionNames);
-        assertThat(expectFieldInResponseMsg, suggestResponse.getSuggest().getSuggestion(name), is(notNullValue()));
+        assertThat(expectFieldInResponseMsg, searchResponse.getSuggest().getSuggestion(name), is(notNullValue()));
 
-        Suggest.Suggestion<Suggest.Suggestion.Entry<Suggest.Suggestion.Entry.Option>> suggestion = suggestResponse.getSuggest().getSuggestion(name);
+        Suggest.Suggestion<Suggest.Suggestion.Entry<Suggest.Suggestion.Entry.Option>> suggestion = searchResponse.getSuggest().getSuggestion(name);
 
         List<String> suggestionList = getNames(suggestion.getEntries().get(0));
         List<Suggest.Suggestion.Entry.Option> options = suggestion.getEntries().get(0).getOptions();
@@ -1122,7 +1110,7 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
             // Exception must be thrown
             assertFalse(true);
         } catch (SearchPhaseExecutionException e) {
-            assertTrue(e.toString().contains("found no fielddata type for field [" + FIELD + "]"));
+            assertThat(e.toString(), containsString("Fielddata is not supported on fields of type [completion]"));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java
index 7096a0f34ac..ee45a9d0371 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java
@@ -19,10 +19,11 @@
 package org.elasticsearch.search.suggest;
 
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
 import org.apache.lucene.spatial.util.GeoHashUtils;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.suggest.SuggestResponse;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
@@ -35,12 +36,14 @@ import org.elasticsearch.search.suggest.completion.context.ContextBuilder;
 import org.elasticsearch.search.suggest.completion.context.ContextMapping;
 import org.elasticsearch.search.suggest.completion.context.GeoContextMapping;
 import org.elasticsearch.search.suggest.completion.context.GeoQueryContext;
+import org.elasticsearch.search.suggest.completion.context.QueryContext;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
@@ -89,7 +92,7 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
         ensureYellow(INDEX);
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg");
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg");
         assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5");
     }
 
@@ -121,7 +124,7 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
         ensureYellow(INDEX);
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).regex("sugg.*es");
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).regex("sugg.*es");
         assertSuggestions("foo", prefix, "sugg9estion", "sugg8estion", "sugg7estion", "sugg6estion", "sugg5estion");
     }
 
@@ -153,7 +156,7 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
         ensureYellow(INDEX);
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg", Fuzziness.ONE);
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg", Fuzziness.ONE);
         assertSuggestions("foo", prefix, "sugxgestion9", "sugxgestion8", "sugxgestion7", "sugxgestion6", "sugxgestion5");
     }
 
@@ -178,8 +181,8 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
         ensureYellow(INDEX);
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg")
-                .categoryContexts("cat", CategoryQueryContext.builder().setCategory("cat0").build());
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg")
+                .contexts(Collections.singletonMap("cat", Collections.singletonList(CategoryQueryContext.builder().setCategory("cat0").build())));
 
         assertSuggestions("foo", prefix, "suggestion8", "suggestion6", "suggestion4", "suggestion2", "suggestion0");
     }
@@ -205,10 +208,10 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
         ensureYellow(INDEX);
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg")
-                .categoryContexts("cat",
-                        CategoryQueryContext.builder().setCategory("cat0").setBoost(3).build(),
-                        CategoryQueryContext.builder().setCategory("cat1").build()
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg")
+                .contexts(Collections.singletonMap("cat",
+                        Arrays.asList(CategoryQueryContext.builder().setCategory("cat0").setBoost(3).build(),
+                        CategoryQueryContext.builder().setCategory("cat1").build()))
                 );
         assertSuggestions("foo", prefix, "suggestion8", "suggestion6", "suggestion4", "suggestion9", "suggestion2");
     }
@@ -235,7 +238,7 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
         ensureYellow(INDEX);
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg");
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg");
 
         assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5");
     }
@@ -265,25 +268,22 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
         ensureYellow(INDEX);
 
         // filter only on context cat
-        CompletionSuggestionBuilder catFilterSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg");
-        catFilterSuggest.categoryContexts("cat", CategoryQueryContext.builder().setCategory("cat0").build());
+        CompletionSuggestionBuilder catFilterSuggest = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg");
+        catFilterSuggest.contexts(Collections.singletonMap("cat", Collections.singletonList(CategoryQueryContext.builder().setCategory("cat0").build())));
         assertSuggestions("foo", catFilterSuggest, "suggestion8", "suggestion6", "suggestion4", "suggestion2", "suggestion0");
 
         // filter only on context type
-        CompletionSuggestionBuilder typeFilterSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg");
-        typeFilterSuggest.categoryContexts("type", CategoryQueryContext.builder().setCategory("type2").build(),
-                CategoryQueryContext.builder().setCategory("type1").build());
+        CompletionSuggestionBuilder typeFilterSuggest = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg");
+        typeFilterSuggest.contexts(Collections.singletonMap("type", Arrays.asList(CategoryQueryContext.builder().setCategory("type2").build(),
+                CategoryQueryContext.builder().setCategory("type1").build())));
         assertSuggestions("foo", typeFilterSuggest, "suggestion9", "suggestion6", "suggestion5", "suggestion2", "suggestion1");
 
-        CompletionSuggestionBuilder multiContextFilterSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg");
+        CompletionSuggestionBuilder multiContextFilterSuggest = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg");
         // query context order should never matter
-        if (randomBoolean()) {
-            multiContextFilterSuggest.categoryContexts("type", CategoryQueryContext.builder().setCategory("type2").build());
-            multiContextFilterSuggest.categoryContexts("cat", CategoryQueryContext.builder().setCategory("cat2").build());
-        } else {
-            multiContextFilterSuggest.categoryContexts("cat", CategoryQueryContext.builder().setCategory("cat2").build());
-            multiContextFilterSuggest.categoryContexts("type", CategoryQueryContext.builder().setCategory("type2").build());
-        }
+        Map<String, List<? extends QueryContext>> contextMap = new HashMap<>();
+        contextMap.put("type", Collections.singletonList(CategoryQueryContext.builder().setCategory("type2").build()));
+        contextMap.put("cat", Collections.singletonList(CategoryQueryContext.builder().setCategory("cat2").build()));
+        multiContextFilterSuggest.contexts(contextMap);
         assertSuggestions("foo", multiContextFilterSuggest, "suggestion6", "suggestion2");
     }
 
@@ -313,37 +313,34 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
         ensureYellow(INDEX);
 
         // boost only on context cat
-        CompletionSuggestionBuilder catBoostSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg");
-        catBoostSuggest.categoryContexts("cat",
-                CategoryQueryContext.builder().setCategory("cat0").setBoost(3).build(),
-                CategoryQueryContext.builder().setCategory("cat1").build());
+        CompletionSuggestionBuilder catBoostSuggest = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg");
+        catBoostSuggest.contexts(Collections.singletonMap("cat",
+                Arrays.asList(
+                    CategoryQueryContext.builder().setCategory("cat0").setBoost(3).build(),
+                    CategoryQueryContext.builder().setCategory("cat1").build())));
         assertSuggestions("foo", catBoostSuggest, "suggestion8", "suggestion6", "suggestion4", "suggestion9", "suggestion2");
 
         // boost only on context type
-        CompletionSuggestionBuilder typeBoostSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg");
-        typeBoostSuggest.categoryContexts("type",
-                CategoryQueryContext.builder().setCategory("type2").setBoost(2).build(),
-                CategoryQueryContext.builder().setCategory("type1").setBoost(4).build());
+        CompletionSuggestionBuilder typeBoostSuggest = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg");
+        typeBoostSuggest.contexts(Collections.singletonMap("type",
+                Arrays.asList(
+                    CategoryQueryContext.builder().setCategory("type2").setBoost(2).build(),
+                    CategoryQueryContext.builder().setCategory("type1").setBoost(4).build())));
         assertSuggestions("foo", typeBoostSuggest, "suggestion9", "suggestion5", "suggestion6", "suggestion1", "suggestion2");
 
         // boost on both contexts
-        CompletionSuggestionBuilder multiContextBoostSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg");
+        CompletionSuggestionBuilder multiContextBoostSuggest = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg");
         // query context order should never matter
-        if (randomBoolean()) {
-            multiContextBoostSuggest.categoryContexts("type",
-                    CategoryQueryContext.builder().setCategory("type2").setBoost(2).build(),
-                    CategoryQueryContext.builder().setCategory("type1").setBoost(4).build());
-            multiContextBoostSuggest.categoryContexts("cat",
-                    CategoryQueryContext.builder().setCategory("cat0").setBoost(3).build(),
-                    CategoryQueryContext.builder().setCategory("cat1").build());
-        } else {
-            multiContextBoostSuggest.categoryContexts("cat",
-                    CategoryQueryContext.builder().setCategory("cat0").setBoost(3).build(),
-                    CategoryQueryContext.builder().setCategory("cat1").build());
-            multiContextBoostSuggest.categoryContexts("type",
-                    CategoryQueryContext.builder().setCategory("type2").setBoost(2).build(),
-                    CategoryQueryContext.builder().setCategory("type1").setBoost(4).build());
-        }
+        Map<String, List<? extends QueryContext>> contextMap = new HashMap<>();
+        contextMap.put("type", Arrays.asList(
+            CategoryQueryContext.builder().setCategory("type2").setBoost(2).build(),
+            CategoryQueryContext.builder().setCategory("type1").setBoost(4).build())
+        );
+        contextMap.put("cat", Arrays.asList(
+            CategoryQueryContext.builder().setCategory("cat0").setBoost(3).build(),
+            CategoryQueryContext.builder().setCategory("cat1").build())
+        );
+        multiContextBoostSuggest.contexts(contextMap);
         assertSuggestions("foo", multiContextBoostSuggest, "suggestion9", "suggestion6", "suggestion5", "suggestion2", "suggestion1");
     }
 
@@ -374,7 +371,7 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
         ensureYellow(INDEX);
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg");
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg");
         assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5");
     }
 
@@ -405,7 +402,7 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
         indexRandom(true, indexRequestBuilders);
         ensureYellow(INDEX);
 
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg");
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg");
         assertSuggestions("foo", prefix, "suggestion0", "suggestion1", "suggestion2", "suggestion3", "suggestion4");
     }
 
@@ -431,7 +428,7 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
         ensureYellow(INDEX);
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg");
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg");
         assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5");
     }
 
@@ -458,11 +455,12 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
         ensureYellow(INDEX);
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg");
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg");
         assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5");
 
-        CompletionSuggestionBuilder geoFilteringPrefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg")
-                .geoContexts("geo", GeoQueryContext.builder().setGeoPoint(new GeoPoint(geoPoints[0])).build());
+        CompletionSuggestionBuilder geoFilteringPrefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg")
+                .contexts(Collections.singletonMap("geo", Collections.singletonList(
+                    GeoQueryContext.builder().setGeoPoint(new GeoPoint(geoPoints[0])).build())));
 
         assertSuggestions("foo", geoFilteringPrefix, "suggestion8", "suggestion6", "suggestion4", "suggestion2", "suggestion0");
     }
@@ -490,13 +488,13 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
         ensureYellow(INDEX);
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg");
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg");
         assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5");
 
         GeoQueryContext context1 = GeoQueryContext.builder().setGeoPoint(geoPoints[0]).setBoost(2).build();
         GeoQueryContext context2 = GeoQueryContext.builder().setGeoPoint(geoPoints[1]).build();
-        CompletionSuggestionBuilder geoBoostingPrefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg")
-                .geoContexts("geo", context1, context2);
+        CompletionSuggestionBuilder geoBoostingPrefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg")
+                .contexts(Collections.singletonMap("geo", Arrays.asList(context1, context2)));
 
         assertSuggestions("foo", geoBoostingPrefix, "suggestion8", "suggestion6", "suggestion4", "suggestion9", "suggestion7");
     }
@@ -526,8 +524,8 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
         ensureYellow(INDEX);
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg")
-                .geoContexts("geo", GeoQueryContext.builder().setGeoPoint(new GeoPoint(52.2263, 4.543)).build());
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg")
+                .contexts(Collections.singletonMap("geo", Collections.singletonList(GeoQueryContext.builder().setGeoPoint(new GeoPoint(52.2263, 4.543)).build())));
         assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5");
     }
 
@@ -564,11 +562,11 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
         ensureYellow(INDEX);
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg");
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg");
         assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5");
 
-        CompletionSuggestionBuilder geoNeighbourPrefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg")
-                .geoContexts("geo", GeoQueryContext.builder().setGeoPoint(GeoPoint.fromGeohash(geohash)).build());
+        CompletionSuggestionBuilder geoNeighbourPrefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg")
+                .contexts(Collections.singletonMap("geo", Collections.singletonList(GeoQueryContext.builder().setGeoPoint(GeoPoint.fromGeohash(geohash)).build())));
 
         assertSuggestions("foo", geoNeighbourPrefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5");
     }
@@ -624,18 +622,19 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase {
         refresh();
 
         String suggestionName = randomAsciiOfLength(10);
-        CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text("h").size(10)
-                .geoContexts("st", GeoQueryContext.builder().setGeoPoint(new GeoPoint(52.52, 13.4)).build());
-        SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(context).get();
+        CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(FIELD).text("h").size(10)
+                .contexts(Collections.singletonMap("st", Collections.singletonList(GeoQueryContext.builder().setGeoPoint(new GeoPoint(52.52, 13.4)).build())));
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion(suggestionName, context)).get();
 
-        assertEquals(suggestResponse.getSuggest().size(), 1);
-        assertEquals("Hotel Amsterdam in Berlin", suggestResponse.getSuggest().getSuggestion(suggestionName).iterator().next().getOptions().iterator().next().getText().string());
+        assertEquals(searchResponse.getSuggest().size(), 1);
+        assertEquals("Hotel Amsterdam in Berlin", searchResponse.getSuggest().getSuggestion(suggestionName).iterator().next().getOptions().iterator().next().getText().string());
     }
 
-    public void assertSuggestions(String suggestionName, SuggestBuilder.SuggestionBuilder suggestBuilder, String... suggestions) {
-        SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestBuilder
+    public void assertSuggestions(String suggestionName, SuggestionBuilder suggestBuilder, String... suggestions) {
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(
+            new SuggestBuilder().addSuggestion(suggestionName, suggestBuilder)
         ).execute().actionGet();
-        CompletionSuggestSearchIT.assertSuggestions(suggestResponse, suggestionName, suggestions);
+        CompletionSuggestSearchIT.assertSuggestions(searchResponse, suggestionName, suggestions);
     }
 
     private void createIndexAndMapping(CompletionMappingBuilder completionMappingBuilder) throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java
index 35d495272ca..150db34ff78 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java
@@ -21,6 +21,7 @@ package org.elasticsearch.search.suggest;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.util.CharsRefBuilder;
 import org.elasticsearch.common.text.Text;
+import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
 import java.util.Locale;
@@ -31,6 +32,7 @@ import java.util.Map;
  */
 public class CustomSuggester extends Suggester<CustomSuggester.CustomSuggestionsContext> {
 
+    public static CustomSuggester PROTOTYPE = new CustomSuggester();
 
     // This is a pretty dumb implementation which returns the original text + fieldName + custom config option + 12 or 123
     @Override
@@ -52,23 +54,18 @@ public class CustomSuggester extends Suggester<CustomSuggester.CustomSuggestions
         return response;
     }
 
-    @Override
-    public SuggestContextParser getContextParser() {
-        return (parser, mapperService, fieldData) -> {
-            Map<String, Object> options = parser.map();
-            CustomSuggestionsContext suggestionContext = new CustomSuggestionsContext(CustomSuggester.this, options);
-            suggestionContext.setField((String) options.get("field"));
-            return suggestionContext;
-        };
-    }
-
     public static class CustomSuggestionsContext extends SuggestionSearchContext.SuggestionContext {
 
         public Map<String, Object> options;
 
-        public CustomSuggestionsContext(Suggester suggester, Map<String, Object> options) {
-            super(suggester);
+        public CustomSuggestionsContext(QueryShardContext context, Map<String, Object> options) {
+            super(new CustomSuggester(), context);
             this.options = options;
         }
     }
+
+    @Override
+    public SuggestionBuilder<?> getBuilderPrototype() {
+        return CustomSuggesterSearchIT.CustomSuggestionBuilder.PROTOTYPE;
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterPlugin.java b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterPlugin.java
index c5e36da2ea7..19d6ed4e098 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterPlugin.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterPlugin.java
@@ -37,7 +37,7 @@ public class CustomSuggesterPlugin extends Plugin {
     }
 
     public void onModule(SearchModule searchModule) {
-        searchModule.registerSuggester("custom", CustomSuggester.class);
+        searchModule.registerSuggester("custom", CustomSuggester.PROTOTYPE);
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
index 18b4fa50e7b..1c82a1aaf37 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
@@ -20,17 +20,30 @@ package org.elasticsearch.search.suggest;
 
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.hasSize;
@@ -59,16 +72,7 @@ public class CustomSuggesterSearchIT extends ESIntegTestCase {
         String randomField = randomAsciiOfLength(10);
         String randomSuffix = randomAsciiOfLength(10);
         SuggestBuilder suggestBuilder = new SuggestBuilder();
-        suggestBuilder.addSuggestion(
-                new SuggestBuilder.SuggestionBuilder<SuggestBuilder.SuggestionBuilder>("someName", "custom") {
-                    @Override
-                    protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
-                        builder.field("field", randomField);
-                        builder.field("suffix", randomSuffix);
-                        return builder;
-                    }
-                }.text(randomText)
-        );
+        suggestBuilder.addSuggestion("someName", new CustomSuggestionBuilder(randomField, randomSuffix).text(randomText));
         SearchRequestBuilder searchRequestBuilder = client().prepareSearch("test").setTypes("test").setFrom(0).setSize(1)
                 .suggest(suggestBuilder);
 
@@ -76,11 +80,121 @@ public class CustomSuggesterSearchIT extends ESIntegTestCase {
 
         // TODO: infer type once JI-9019884 is fixed
         // TODO: see also JDK-8039214
-        List<Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>> suggestions
-                = CollectionUtils.<Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>iterableAsArrayList(searchResponse.getSuggest().getSuggestion("someName"));
+        List<Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>> suggestions =
+            CollectionUtils.<Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>iterableAsArrayList(
+                searchResponse.getSuggest().getSuggestion("someName"));
         assertThat(suggestions, hasSize(2));
-        assertThat(suggestions.get(0).getText().string(), is(String.format(Locale.ROOT, "%s-%s-%s-12", randomText, randomField, randomSuffix)));
-        assertThat(suggestions.get(1).getText().string(), is(String.format(Locale.ROOT, "%s-%s-%s-123", randomText, randomField, randomSuffix)));
+        assertThat(suggestions.get(0).getText().string(),
+            is(String.format(Locale.ROOT, "%s-%s-%s-12", randomText, randomField, randomSuffix)));
+        assertThat(suggestions.get(1).getText().string(),
+            is(String.format(Locale.ROOT, "%s-%s-%s-123", randomText, randomField, randomSuffix)));
+    }
+
+    public static class CustomSuggestionBuilder extends SuggestionBuilder<CustomSuggestionBuilder> {
+
+        public final static CustomSuggestionBuilder PROTOTYPE = new CustomSuggestionBuilder("_na_", "_na_");
+        protected static final ParseField RANDOM_SUFFIX_FIELD = new ParseField("suffix");
+
+        private String randomSuffix;
+
+        public CustomSuggestionBuilder(String randomField, String randomSuffix) {
+            super(randomField);
+            this.randomSuffix = randomSuffix;
+        }
+
+        @Override
+        protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.field(RANDOM_SUFFIX_FIELD.getPreferredName(), randomSuffix);
+            return builder;
+        }
+
+        @Override
+        public String getWriteableName() {
+            return "custom";
+        }
+
+        @Override
+        public void doWriteTo(StreamOutput out) throws IOException {
+            out.writeString(randomSuffix);
+        }
+
+        @Override
+        public CustomSuggestionBuilder doReadFrom(StreamInput in, String field) throws IOException {
+            return new CustomSuggestionBuilder(field, in.readString());
+        }
+
+        @Override
+        protected boolean doEquals(CustomSuggestionBuilder other) {
+            return Objects.equals(randomSuffix, other.randomSuffix);
+        }
+
+        @Override
+        protected int doHashCode() {
+            return Objects.hash(randomSuffix);
+        }
+
+        @Override
+        protected CustomSuggestionBuilder innerFromXContent(QueryParseContext parseContext) throws IOException {
+            XContentParser parser = parseContext.parser();
+            ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher();
+            XContentParser.Token token;
+            String currentFieldName = null;
+            String fieldname = null;
+            String suffix = null;
+            String analyzer = null;
+            int sizeField = -1;
+            int shardSize = -1;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (token.isValue()) {
+                    if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.ANALYZER_FIELD)) {
+                        analyzer = parser.text();
+                    } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.FIELDNAME_FIELD)) {
+                        fieldname = parser.text();
+                    } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SIZE_FIELD)) {
+                        sizeField = parser.intValue();
+                    } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SHARDSIZE_FIELD)) {
+                        shardSize = parser.intValue();
+                    } else if (parseFieldMatcher.match(currentFieldName, RANDOM_SUFFIX_FIELD)) {
+                        suffix = parser.text();
+                    }
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(),
+                                               "suggester[custom] doesn't support field [" + currentFieldName + "]");
+                }
+            }
+
+            // now we should have field name, check and copy fields over to the suggestion builder we return
+            if (fieldname == null) {
+                throw new ParsingException(parser.getTokenLocation(), "the required field option is missing");
+            }
+            CustomSuggestionBuilder builder = new CustomSuggestionBuilder(fieldname, suffix);
+            if (analyzer != null) {
+                builder.analyzer(analyzer);
+            }
+            if (sizeField != -1) {
+                builder.size(sizeField);
+            }
+            if (shardSize != -1) {
+                builder.shardSize(shardSize);
+            }
+            return builder;
+        }
+
+        @Override
+        public SuggestionContext build(QueryShardContext context) throws IOException {
+            Map<String, Object> options = new HashMap<>();
+            options.put(FIELDNAME_FIELD.getPreferredName(), field());
+            options.put(RANDOM_SUFFIX_FIELD.getPreferredName(), randomSuffix);
+            CustomSuggester.CustomSuggestionsContext customSuggestionsContext =
+                new CustomSuggester.CustomSuggestionsContext(context, options);
+            customSuggestionsContext.setField(field());
+            assert text != null;
+            customSuggestionsContext.setText(BytesRefs.toBytesRef(text));
+            return customSuggestionsContext;
+        }
+
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java
new file mode 100644
index 00000000000..ea3ff5c7477
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest;
+
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.suggest.completion.CompletionSuggesterBuilderTests;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
+import org.elasticsearch.search.suggest.completion.WritableTestCase;
+import org.elasticsearch.search.suggest.phrase.Laplace;
+import org.elasticsearch.search.suggest.phrase.LinearInterpolation;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilderTests;
+import org.elasticsearch.search.suggest.phrase.SmoothingModel;
+import org.elasticsearch.search.suggest.phrase.StupidBackoff;
+import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
+import org.elasticsearch.search.suggest.term.TermSuggestionBuilderTests;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map.Entry;
+
+public class SuggestBuilderTests extends WritableTestCase<SuggestBuilder> {
+
+    private static NamedWriteableRegistry namedWriteableRegistry;
+
+    /**
+     * Setup for the whole base test class.
+     */
+    @BeforeClass
+    public static void init() {
+        NamedWriteableRegistry nwRegistry = new NamedWriteableRegistry();
+        nwRegistry.registerPrototype(SuggestionBuilder.class, TermSuggestionBuilder.PROTOTYPE);
+        nwRegistry.registerPrototype(SuggestionBuilder.class, PhraseSuggestionBuilder.PROTOTYPE);
+        nwRegistry.registerPrototype(SuggestionBuilder.class, CompletionSuggestionBuilder.PROTOTYPE);
+        nwRegistry.registerPrototype(SmoothingModel.class, Laplace.PROTOTYPE);
+        nwRegistry.registerPrototype(SmoothingModel.class, LinearInterpolation.PROTOTYPE);
+        nwRegistry.registerPrototype(SmoothingModel.class, StupidBackoff.PROTOTYPE);
+        namedWriteableRegistry = nwRegistry;
+    }
+
+    @AfterClass
+    public static void afterClass() {
+        namedWriteableRegistry = null;
+    }
+
+    @Override
+    protected NamedWriteableRegistry provideNamedWritableRegistry() {
+        return namedWriteableRegistry;
+    }
+
+    /**
+     *  creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original
+     */
+    public void testFromXContent() throws IOException {
+        Suggesters suggesters = new Suggesters(Collections.emptyMap());
+        QueryParseContext context = new QueryParseContext(null);
+        context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY));
+        for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) {
+            SuggestBuilder suggestBuilder = createTestModel();
+            XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+            if (randomBoolean()) {
+                xContentBuilder.prettyPrint();
+            }
+            suggestBuilder.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
+            XContentParser parser = XContentHelper.createParser(xContentBuilder.bytes());
+            context.reset(parser);
+
+            SuggestBuilder secondSuggestBuilder = SuggestBuilder.fromXContent(context, suggesters);
+            assertNotSame(suggestBuilder, secondSuggestBuilder);
+            assertEquals(suggestBuilder, secondSuggestBuilder);
+            assertEquals(suggestBuilder.hashCode(), secondSuggestBuilder.hashCode());
+        }
+    }
+
+    public void testIllegalSuggestionName() {
+        try {
+            new SuggestBuilder().addSuggestion(null, PhraseSuggestionBuilderTests.randomPhraseSuggestionBuilder());
+            fail("exception expected");
+        } catch (NullPointerException e) {
+            assertEquals("every suggestion needs a name", e.getMessage());
+        }
+
+        try {
+            new SuggestBuilder().addSuggestion("my-suggest", PhraseSuggestionBuilderTests.randomPhraseSuggestionBuilder())
+                .addSuggestion("my-suggest", PhraseSuggestionBuilderTests.randomPhraseSuggestionBuilder());
+            fail("exception expected");
+        } catch (IllegalArgumentException e) {
+            assertEquals("already added another suggestion with name [my-suggest]", e.getMessage());
+        }
+    }
+
+    @Override
+    protected SuggestBuilder createTestModel() {
+        return randomSuggestBuilder();
+    }
+
+    @Override
+    protected SuggestBuilder createMutation(SuggestBuilder original) throws IOException {
+        SuggestBuilder mutation = new SuggestBuilder().setGlobalText(original.getGlobalText());
+        for (Entry<String, SuggestionBuilder<?>> suggestionBuilder : original.getSuggestions().entrySet()) {
+            mutation.addSuggestion(suggestionBuilder.getKey(), suggestionBuilder.getValue());
+        }
+        if (randomBoolean()) {
+            mutation.setGlobalText(randomAsciiOfLengthBetween(5, 60));
+        } else {
+            mutation.addSuggestion(randomAsciiOfLength(10), PhraseSuggestionBuilderTests.randomPhraseSuggestionBuilder());
+        }
+        return mutation;
+    }
+
+    @Override
+    protected SuggestBuilder readFrom(StreamInput in) throws IOException {
+        return SuggestBuilder.PROTOTYPE.readFrom(in);
+    }
+
+    public static SuggestBuilder randomSuggestBuilder() {
+        SuggestBuilder builder = new SuggestBuilder();
+        if (randomBoolean()) {
+            builder.setGlobalText(randomAsciiOfLengthBetween(1, 20));
+        }
+        final int numSuggestions = randomIntBetween(1, 5);
+        for (int i = 0; i < numSuggestions; i++) {
+            builder.addSuggestion(randomAsciiOfLengthBetween(5, 10), randomSuggestionBuilder());
+        }
+        return builder;
+    }
+
+    private static SuggestionBuilder<?> randomSuggestionBuilder() {
+        switch (randomIntBetween(0, 2)) {
+            case 0: return TermSuggestionBuilderTests.randomTermSuggestionBuilder();
+            case 1: return PhraseSuggestionBuilderTests.randomPhraseSuggestionBuilder();
+            case 2: return CompletionSuggesterBuilderTests.randomCompletionSuggestionBuilder();
+            default: return TermSuggestionBuilderTests.randomTermSuggestionBuilder();
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java
index 0d27ba04a91..e0aabe77302 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java
@@ -190,11 +190,11 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
         XContentBuilder builder = jsonBuilder().value("context1");
         XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
         CategoryContextMapping mapping = ContextBuilder.category("cat").build();
-        List<ContextMapping.QueryContext> queryContexts = mapping.parseQueryContext(parser);
-        assertThat(queryContexts.size(), equalTo(1));
-        assertThat(queryContexts.get(0).context, equalTo("context1"));
-        assertThat(queryContexts.get(0).boost, equalTo(1));
-        assertThat(queryContexts.get(0).isPrefix, equalTo(false));
+        List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(parser);
+        assertThat(internalQueryContexts.size(), equalTo(1));
+        assertThat(internalQueryContexts.get(0).context, equalTo("context1"));
+        assertThat(internalQueryContexts.get(0).boost, equalTo(1));
+        assertThat(internalQueryContexts.get(0).isPrefix, equalTo(false));
     }
 
     public void testQueryContextParsingArray() throws Exception {
@@ -204,14 +204,14 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                 .endArray();
         XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
         CategoryContextMapping mapping = ContextBuilder.category("cat").build();
-        List<ContextMapping.QueryContext> queryContexts = mapping.parseQueryContext(parser);
-        assertThat(queryContexts.size(), equalTo(2));
-        assertThat(queryContexts.get(0).context, equalTo("context1"));
-        assertThat(queryContexts.get(0).boost, equalTo(1));
-        assertThat(queryContexts.get(0).isPrefix, equalTo(false));
-        assertThat(queryContexts.get(1).context, equalTo("context2"));
-        assertThat(queryContexts.get(1).boost, equalTo(1));
-        assertThat(queryContexts.get(1).isPrefix, equalTo(false));
+        List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(parser);
+        assertThat(internalQueryContexts.size(), equalTo(2));
+        assertThat(internalQueryContexts.get(0).context, equalTo("context1"));
+        assertThat(internalQueryContexts.get(0).boost, equalTo(1));
+        assertThat(internalQueryContexts.get(0).isPrefix, equalTo(false));
+        assertThat(internalQueryContexts.get(1).context, equalTo("context2"));
+        assertThat(internalQueryContexts.get(1).boost, equalTo(1));
+        assertThat(internalQueryContexts.get(1).isPrefix, equalTo(false));
     }
 
     public void testQueryContextParsingObject() throws Exception {
@@ -222,11 +222,11 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                 .endObject();
         XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
         CategoryContextMapping mapping = ContextBuilder.category("cat").build();
-        List<ContextMapping.QueryContext> queryContexts = mapping.parseQueryContext(parser);
-        assertThat(queryContexts.size(), equalTo(1));
-        assertThat(queryContexts.get(0).context, equalTo("context1"));
-        assertThat(queryContexts.get(0).boost, equalTo(10));
-        assertThat(queryContexts.get(0).isPrefix, equalTo(true));
+        List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(parser);
+        assertThat(internalQueryContexts.size(), equalTo(1));
+        assertThat(internalQueryContexts.get(0).context, equalTo("context1"));
+        assertThat(internalQueryContexts.get(0).boost, equalTo(10));
+        assertThat(internalQueryContexts.get(0).isPrefix, equalTo(true));
     }
 
 
@@ -245,14 +245,14 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                 .endArray();
         XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
         CategoryContextMapping mapping = ContextBuilder.category("cat").build();
-        List<ContextMapping.QueryContext> queryContexts = mapping.parseQueryContext(parser);
-        assertThat(queryContexts.size(), equalTo(2));
-        assertThat(queryContexts.get(0).context, equalTo("context1"));
-        assertThat(queryContexts.get(0).boost, equalTo(2));
-        assertThat(queryContexts.get(0).isPrefix, equalTo(true));
-        assertThat(queryContexts.get(1).context, equalTo("context2"));
-        assertThat(queryContexts.get(1).boost, equalTo(3));
-        assertThat(queryContexts.get(1).isPrefix, equalTo(false));
+        List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(parser);
+        assertThat(internalQueryContexts.size(), equalTo(2));
+        assertThat(internalQueryContexts.get(0).context, equalTo("context1"));
+        assertThat(internalQueryContexts.get(0).boost, equalTo(2));
+        assertThat(internalQueryContexts.get(0).isPrefix, equalTo(true));
+        assertThat(internalQueryContexts.get(1).context, equalTo("context2"));
+        assertThat(internalQueryContexts.get(1).boost, equalTo(3));
+        assertThat(internalQueryContexts.get(1).isPrefix, equalTo(false));
     }
 
     public void testQueryContextParsingMixed() throws Exception {
@@ -266,14 +266,14 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                 .endArray();
         XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
         CategoryContextMapping mapping = ContextBuilder.category("cat").build();
-        List<ContextMapping.QueryContext> queryContexts = mapping.parseQueryContext(parser);
-        assertThat(queryContexts.size(), equalTo(2));
-        assertThat(queryContexts.get(0).context, equalTo("context1"));
-        assertThat(queryContexts.get(0).boost, equalTo(2));
-        assertThat(queryContexts.get(0).isPrefix, equalTo(true));
-        assertThat(queryContexts.get(1).context, equalTo("context2"));
-        assertThat(queryContexts.get(1).boost, equalTo(1));
-        assertThat(queryContexts.get(1).isPrefix, equalTo(false));
+        List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(parser);
+        assertThat(internalQueryContexts.size(), equalTo(2));
+        assertThat(internalQueryContexts.get(0).context, equalTo("context1"));
+        assertThat(internalQueryContexts.get(0).boost, equalTo(2));
+        assertThat(internalQueryContexts.get(0).isPrefix, equalTo(true));
+        assertThat(internalQueryContexts.get(1).context, equalTo("context2"));
+        assertThat(internalQueryContexts.get(1).boost, equalTo(1));
+        assertThat(internalQueryContexts.get(1).isPrefix, equalTo(false));
     }
 
     public void testParsingContextFromDocument() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryQueryContextTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryQueryContextTests.java
new file mode 100644
index 00000000000..a4cfc71a3bc
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryQueryContextTests.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.completion;
+
+import org.elasticsearch.search.suggest.completion.context.CategoryQueryContext;
+
+import java.io.IOException;
+
+public class CategoryQueryContextTests extends QueryContextTestCase<CategoryQueryContext> {
+
+    public static CategoryQueryContext randomCategoryQueryContext() {
+        final CategoryQueryContext.Builder builder = CategoryQueryContext.builder();
+        builder.setCategory(randomAsciiOfLength(10));
+        maybeSet(builder::setBoost, randomIntBetween(1, 10));
+        maybeSet(builder::setPrefix, randomBoolean());
+        return builder.build();
+    }
+
+    @Override
+    protected CategoryQueryContext createTestModel() {
+        return randomCategoryQueryContext();
+    }
+
+    @Override
+    protected CategoryQueryContext prototype() {
+        return CategoryQueryContext.PROTOTYPE;
+    }
+
+    public void testNullCategoryIsIllegal() {
+        final CategoryQueryContext categoryQueryContext = randomCategoryQueryContext();
+        final CategoryQueryContext.Builder builder = CategoryQueryContext.builder()
+            .setBoost(categoryQueryContext.getBoost())
+            .setPrefix(categoryQueryContext.isPrefix());
+        try {
+            builder.build();
+            fail("null category is illegal");
+        } catch (NullPointerException e) {
+            assertEquals(e.getMessage(), "category must not be null");
+        }
+    }
+
+    public void testIllegalArguments() {
+        final CategoryQueryContext.Builder builder = CategoryQueryContext.builder();
+
+        try {
+            builder.setCategory(null);
+            fail("category must not be null");
+        } catch (NullPointerException e) {
+            assertEquals(e.getMessage(), "category must not be null");
+        }
+
+        try {
+            builder.setBoost(-randomIntBetween(1, Integer.MAX_VALUE));
+            fail("boost must be positive");
+        } catch (IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "boost must be greater than 0");
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java
new file mode 100644
index 00000000000..6551f2370a7
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.completion;
+
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
+import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase;
+import org.elasticsearch.search.suggest.SuggestBuilder;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
+import org.elasticsearch.search.suggest.completion.context.CategoryContextMapping;
+import org.elasticsearch.search.suggest.completion.context.CategoryQueryContext;
+import org.elasticsearch.search.suggest.completion.context.ContextMapping;
+import org.elasticsearch.search.suggest.completion.context.ContextMappings;
+import org.elasticsearch.search.suggest.completion.context.GeoContextMapping;
+import org.elasticsearch.search.suggest.completion.context.GeoQueryContext;
+import org.elasticsearch.search.suggest.completion.context.QueryContext;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.core.IsInstanceOf.instanceOf;
+import static org.hamcrest.Matchers.containsString;
+
+public class CompletionSuggesterBuilderTests extends AbstractSuggestionBuilderTestCase<CompletionSuggestionBuilder> {
+
+    @Override
+    protected CompletionSuggestionBuilder randomSuggestionBuilder() {
+        return randomCompletionSuggestionBuilder();
+    }
+
+    public static CompletionSuggestionBuilder randomCompletionSuggestionBuilder() {
+        return randomSuggestionBuilderWithContextInfo().builder;
+    }
+
+    private static class BuilderAndInfo {
+        CompletionSuggestionBuilder builder;
+        List<String> catContexts = new ArrayList<>();
+        List<String> geoContexts = new ArrayList<>();
+    }
+
+    private static BuilderAndInfo randomSuggestionBuilderWithContextInfo() {
+        final BuilderAndInfo builderAndInfo = new BuilderAndInfo();
+        CompletionSuggestionBuilder testBuilder = new CompletionSuggestionBuilder(randomAsciiOfLengthBetween(2, 20));
+        setCommonPropertiesOnRandomBuilder(testBuilder);
+        switch (randomIntBetween(0, 3)) {
+            case 0:
+                testBuilder.prefix(randomAsciiOfLength(10));
+                break;
+            case 1:
+                testBuilder.prefix(randomAsciiOfLength(10), FuzzyOptionsTests.randomFuzzyOptions());
+                break;
+            case 2:
+                testBuilder.prefix(randomAsciiOfLength(10), randomFrom(Fuzziness.ZERO, Fuzziness.ONE, Fuzziness.TWO));
+                break;
+            case 3:
+                testBuilder.regex(randomAsciiOfLength(10), RegexOptionsTests.randomRegexOptions());
+                break;
+        }
+        List<String> payloads = new ArrayList<>();
+        Collections.addAll(payloads, generateRandomStringArray(5, 10, false, false));
+        maybeSet(testBuilder::payload, payloads);
+        Map<String, List<? extends QueryContext>> contextMap = new HashMap<>();
+        if (randomBoolean()) {
+            int numContext = randomIntBetween(1, 5);
+            List<CategoryQueryContext> contexts = new ArrayList<>(numContext);
+            for (int i = 0; i < numContext; i++) {
+                contexts.add(CategoryQueryContextTests.randomCategoryQueryContext());
+            }
+            String name = randomAsciiOfLength(10);
+            contextMap.put(name, contexts);
+            builderAndInfo.catContexts.add(name);
+        }
+        if (randomBoolean()) {
+            int numContext = randomIntBetween(1, 5);
+            List<GeoQueryContext> contexts = new ArrayList<>(numContext);
+            for (int i = 0; i < numContext; i++) {
+                contexts.add(GeoQueryContextTests.randomGeoQueryContext());
+            }
+            String name = randomAsciiOfLength(10);
+            contextMap.put(name, contexts);
+            builderAndInfo.geoContexts.add(name);
+        }
+        testBuilder.contexts(contextMap);
+        builderAndInfo.builder = testBuilder;
+        return builderAndInfo;
+    }
+
+    @Override
+    protected void mutateSpecificParameters(CompletionSuggestionBuilder builder) throws IOException {
+        switch (randomIntBetween(0, 5)) {
+            case 0:
+                List<String> payloads = new ArrayList<>();
+                Collections.addAll(payloads, generateRandomStringArray(5, 10, false, false));
+                builder.payload(payloads);
+                break;
+            case 1:
+                int nCatContext = randomIntBetween(1, 5);
+                List<CategoryQueryContext> contexts = new ArrayList<>(nCatContext);
+                for (int i = 0; i < nCatContext; i++) {
+                    contexts.add(CategoryQueryContextTests.randomCategoryQueryContext());
+                }
+                builder.contexts(Collections.singletonMap(randomAsciiOfLength(10), contexts));
+                break;
+            case 2:
+                int nGeoContext = randomIntBetween(1, 5);
+                List<GeoQueryContext> geoContexts = new ArrayList<>(nGeoContext);
+                for (int i = 0; i < nGeoContext; i++) {
+                    geoContexts.add(GeoQueryContextTests.randomGeoQueryContext());
+                }
+                builder.contexts(Collections.singletonMap(randomAsciiOfLength(10), geoContexts));
+                break;
+            case 3:
+                builder.prefix(randomAsciiOfLength(10), FuzzyOptionsTests.randomFuzzyOptions());
+                break;
+            case 4:
+                builder.prefix(randomAsciiOfLength(10), randomFrom(Fuzziness.ZERO, Fuzziness.ONE, Fuzziness.TWO));
+                break;
+            case 5:
+                builder.regex(randomAsciiOfLength(10), RegexOptionsTests.randomRegexOptions());
+                break;
+            default:
+                throw new IllegalStateException("should not through");
+        }
+    }
+
+    /**
+     * Test that a malformed JSON suggestion request fails.
+     */
+    public void testMalformedJsonRequestPayload() throws Exception {
+        final String field = RandomStrings.randomAsciiOfLength(getRandom(), 10).toLowerCase(Locale.ROOT);
+        final String payload = "{\n" +
+                               "  \"bad-payload\" : { \n" +
+                               "    \"prefix\" : \"sug\",\n" +
+                               "    \"completion\" : { \n" +
+                               "      \"field\" : \"" + field + "\",\n " +
+                               "      \"payload\" : [ {\"payload\":\"field\"} ]\n" +
+                               "    }\n" +
+                               "  }\n" +
+                               "}\n";
+        try {
+            final SuggestBuilder suggestBuilder = SuggestBuilder.fromXContent(newParseContext(payload), suggesters);
+            fail("Should not have been able to create SuggestBuilder from malformed JSON: " + suggestBuilder);
+        } catch (ParsingException e) {
+            assertThat(e.getMessage(), containsString("failed to parse field [payload]"));
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/FuzzyOptionsTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/FuzzyOptionsTests.java
new file mode 100644
index 00000000000..848a9088bc3
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/FuzzyOptionsTests.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.completion;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.unit.Fuzziness;
+
+import java.io.IOException;
+
+public class FuzzyOptionsTests extends WritableTestCase<FuzzyOptions> {
+
+    public static FuzzyOptions randomFuzzyOptions() {
+        final FuzzyOptions.Builder builder = FuzzyOptions.builder();
+        if (randomBoolean()) {
+            maybeSet(builder::setFuzziness, randomFrom(Fuzziness.ZERO, Fuzziness.ONE, Fuzziness.TWO));
+        } else {
+            maybeSet(builder::setFuzziness, randomFrom(0, 1, 2));
+        }
+        maybeSet(builder::setFuzzyMinLength, randomIntBetween(0, 10));
+        maybeSet(builder::setFuzzyPrefixLength, randomIntBetween(0, 10));
+        maybeSet(builder::setMaxDeterminizedStates, randomIntBetween(1, 1000));
+        maybeSet(builder::setTranspositions, randomBoolean());
+        maybeSet(builder::setUnicodeAware, randomBoolean());
+        return builder.build();
+    }
+
+    @Override
+    protected FuzzyOptions createTestModel() {
+        return randomFuzzyOptions();
+    }
+
+    @Override
+    protected FuzzyOptions createMutation(FuzzyOptions original) throws IOException {
+        final FuzzyOptions.Builder builder = FuzzyOptions.builder();
+        builder.setFuzziness(original.getEditDistance())
+            .setFuzzyPrefixLength(original.getFuzzyPrefixLength())
+            .setFuzzyMinLength(original.getFuzzyMinLength())
+            .setMaxDeterminizedStates(original.getMaxDeterminizedStates())
+            .setTranspositions(original.isTranspositions())
+            .setUnicodeAware(original.isUnicodeAware());
+        switch (randomIntBetween(0, 5)) {
+            case 0:
+                builder.setFuzziness(randomValueOtherThan(original.getEditDistance(), () -> randomFrom(0, 1, 2)));
+                break;
+            case 1:
+                builder.setFuzzyPrefixLength(randomValueOtherThan(original.getFuzzyPrefixLength(), () ->
+                    randomIntBetween(1, 3)));
+                break;
+            case 2:
+                builder.setFuzzyMinLength(randomValueOtherThan(original.getFuzzyMinLength(), () ->
+                    randomIntBetween(1, 3)));
+                break;
+            case 3:
+                builder.setMaxDeterminizedStates(randomValueOtherThan(original.getMaxDeterminizedStates(), () ->
+                    randomIntBetween(1, 10)));
+                break;
+            case 4:
+                builder.setTranspositions(!original.isTranspositions());
+                break;
+            case 5:
+                builder.setUnicodeAware(!original.isUnicodeAware());
+                break;
+        }
+        return builder.build();
+    }
+
+    @Override
+    protected FuzzyOptions readFrom(StreamInput in) throws IOException {
+        return FuzzyOptions.readFuzzyOptions(in);
+    }
+
+    public void testIllegalArguments() {
+        final FuzzyOptions.Builder builder = FuzzyOptions.builder();
+        try {
+            builder.setFuzziness(-randomIntBetween(1, Integer.MAX_VALUE));
+            fail("fuzziness must be > 0");
+        } catch (IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "fuzziness must be between 0 and 2");
+        }
+        try {
+            builder.setFuzziness(randomIntBetween(3, Integer.MAX_VALUE));
+            fail("fuzziness must be < 2");
+        } catch (IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "fuzziness must be between 0 and 2");
+        }
+        try {
+            builder.setFuzziness(null);
+            fail("fuzziness must not be null");
+        } catch (NullPointerException e) {
+            assertEquals(e.getMessage(), "fuzziness must not be null");
+        }
+
+        try {
+            builder.setFuzzyMinLength(-randomIntBetween(1, Integer.MAX_VALUE));
+            fail("fuzzyMinLength must be >= 0");
+        } catch (IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "fuzzyMinLength must not be negative");
+        }
+
+        try {
+            builder.setFuzzyPrefixLength(-randomIntBetween(1, Integer.MAX_VALUE));
+            fail("fuzzyPrefixLength must be >= 0");
+        } catch (IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "fuzzyPrefixLength must not be negative");
+        }
+
+        try {
+            builder.setMaxDeterminizedStates(-randomIntBetween(1, Integer.MAX_VALUE));
+            fail("max determinized state must be >= 0");
+        } catch (IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "maxDeterminizedStates must not be negative");
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java
index 471de9c3e93..c16b0ce645b 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java
@@ -202,15 +202,15 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
         XContentBuilder builder = jsonBuilder().value("ezs42e44yx96");
         XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
         GeoContextMapping mapping = ContextBuilder.geo("geo").build();
-        List<ContextMapping.QueryContext> queryContexts = mapping.parseQueryContext(parser);
-        assertThat(queryContexts.size(), equalTo(1 + 8));
+        List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(parser);
+        assertThat(internalQueryContexts.size(), equalTo(1 + 8));
         Collection<String> locations = new ArrayList<>();
         locations.add("ezs42e");
         addNeighbors("ezs42e", GeoContextMapping.DEFAULT_PRECISION, locations);
-        for (ContextMapping.QueryContext queryContext : queryContexts) {
-            assertThat(queryContext.context, isIn(locations));
-            assertThat(queryContext.boost, equalTo(1));
-            assertThat(queryContext.isPrefix, equalTo(false));
+        for (ContextMapping.InternalQueryContext internalQueryContext : internalQueryContexts) {
+            assertThat(internalQueryContext.context, isIn(locations));
+            assertThat(internalQueryContext.boost, equalTo(1));
+            assertThat(internalQueryContext.isPrefix, equalTo(false));
         }
     }
 
@@ -221,15 +221,15 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
                 .endObject();
         XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
         GeoContextMapping mapping = ContextBuilder.geo("geo").build();
-        List<ContextMapping.QueryContext> queryContexts = mapping.parseQueryContext(parser);
-        assertThat(queryContexts.size(), equalTo(1 + 8));
+        List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(parser);
+        assertThat(internalQueryContexts.size(), equalTo(1 + 8));
         Collection<String> locations = new ArrayList<>();
         locations.add("wh0n94");
         addNeighbors("wh0n94", GeoContextMapping.DEFAULT_PRECISION, locations);
-        for (ContextMapping.QueryContext queryContext : queryContexts) {
-            assertThat(queryContext.context, isIn(locations));
-            assertThat(queryContext.boost, equalTo(1));
-            assertThat(queryContext.isPrefix, equalTo(false));
+        for (ContextMapping.InternalQueryContext internalQueryContext : internalQueryContexts) {
+            assertThat(internalQueryContext.context, isIn(locations));
+            assertThat(internalQueryContext.boost, equalTo(1));
+            assertThat(internalQueryContext.isPrefix, equalTo(false));
         }
     }
 
@@ -244,8 +244,8 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
                 .endObject();
         XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
         GeoContextMapping mapping = ContextBuilder.geo("geo").build();
-        List<ContextMapping.QueryContext> queryContexts = mapping.parseQueryContext(parser);
-        assertThat(queryContexts.size(), equalTo(1 + 1 + 8 + 1 + 8 + 1 + 8));
+        List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(parser);
+        assertThat(internalQueryContexts.size(), equalTo(1 + 1 + 8 + 1 + 8 + 1 + 8));
         Collection<String> locations = new ArrayList<>();
         locations.add("wh0n94");
         locations.add("w");
@@ -254,10 +254,10 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
         addNeighbors("wh", 2, locations);
         locations.add("wh0");
         addNeighbors("wh0", 3, locations);
-        for (ContextMapping.QueryContext queryContext : queryContexts) {
-            assertThat(queryContext.context, isIn(locations));
-            assertThat(queryContext.boost, equalTo(10));
-            assertThat(queryContext.isPrefix, equalTo(queryContext.context.length() < GeoContextMapping.DEFAULT_PRECISION));
+        for (ContextMapping.InternalQueryContext internalQueryContext : internalQueryContexts) {
+            assertThat(internalQueryContext.context, isIn(locations));
+            assertThat(internalQueryContext.boost, equalTo(10));
+            assertThat(internalQueryContext.isPrefix, equalTo(internalQueryContext.context.length() < GeoContextMapping.DEFAULT_PRECISION));
         }
     }
 
@@ -282,8 +282,8 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
                 .endArray();
         XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
         GeoContextMapping mapping = ContextBuilder.geo("geo").build();
-        List<ContextMapping.QueryContext> queryContexts = mapping.parseQueryContext(parser);
-        assertThat(queryContexts.size(), equalTo(1 + 1 + 8 + 1 + 8 + 1 + 8 + 1 + 1 + 8));
+        List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(parser);
+        assertThat(internalQueryContexts.size(), equalTo(1 + 1 + 8 + 1 + 8 + 1 + 8 + 1 + 1 + 8));
         Collection<String> firstLocations = new ArrayList<>();
         firstLocations.add("wh0n94");
         firstLocations.add("w");
@@ -296,15 +296,15 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
         secondLocations.add("w5cx04");
         secondLocations.add("w5cx0");
         addNeighbors("w5cx0", 5, secondLocations);
-        for (ContextMapping.QueryContext queryContext : queryContexts) {
-            if (firstLocations.contains(queryContext.context)) {
-                assertThat(queryContext.boost, equalTo(10));
-            } else if (secondLocations.contains(queryContext.context)) {
-                assertThat(queryContext.boost, equalTo(2));
+        for (ContextMapping.InternalQueryContext internalQueryContext : internalQueryContexts) {
+            if (firstLocations.contains(internalQueryContext.context)) {
+                assertThat(internalQueryContext.boost, equalTo(10));
+            } else if (secondLocations.contains(internalQueryContext.context)) {
+                assertThat(internalQueryContext.boost, equalTo(2));
             } else {
-                fail(queryContext.context + " was not expected");
+                fail(internalQueryContext.context + " was not expected");
             }
-            assertThat(queryContext.isPrefix, equalTo(queryContext.context.length() < GeoContextMapping.DEFAULT_PRECISION));
+            assertThat(internalQueryContext.isPrefix, equalTo(internalQueryContext.context.length() < GeoContextMapping.DEFAULT_PRECISION));
         }
     }
 
@@ -325,8 +325,8 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
                 .endArray();
         XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes());
         GeoContextMapping mapping = ContextBuilder.geo("geo").build();
-        List<ContextMapping.QueryContext> queryContexts = mapping.parseQueryContext(parser);
-        assertThat(queryContexts.size(), equalTo(1 + 1 + 8 + 1 + 8 + 1 + 8));
+        List<ContextMapping.InternalQueryContext> internalQueryContexts = mapping.parseQueryContext(parser);
+        assertThat(internalQueryContexts.size(), equalTo(1 + 1 + 8 + 1 + 8 + 1 + 8));
         Collection<String> firstLocations = new ArrayList<>();
         firstLocations.add("wh0n94");
         firstLocations.add("w");
@@ -336,15 +336,15 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
         Collection<String> secondLocations = new ArrayList<>();
         secondLocations.add("w5cx04");
         addNeighbors("w5cx04", 6, secondLocations);
-        for (ContextMapping.QueryContext queryContext : queryContexts) {
-            if (firstLocations.contains(queryContext.context)) {
-                assertThat(queryContext.boost, equalTo(10));
-            } else if (secondLocations.contains(queryContext.context)) {
-                assertThat(queryContext.boost, equalTo(1));
+        for (ContextMapping.InternalQueryContext internalQueryContext : internalQueryContexts) {
+            if (firstLocations.contains(internalQueryContext.context)) {
+                assertThat(internalQueryContext.boost, equalTo(10));
+            } else if (secondLocations.contains(internalQueryContext.context)) {
+                assertThat(internalQueryContext.boost, equalTo(1));
             } else {
-                fail(queryContext.context + " was not expected");
+                fail(internalQueryContext.context + " was not expected");
             }
-            assertThat(queryContext.isPrefix, equalTo(queryContext.context.length() < GeoContextMapping.DEFAULT_PRECISION));
+            assertThat(internalQueryContext.isPrefix, equalTo(internalQueryContext.context.length() < GeoContextMapping.DEFAULT_PRECISION));
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoQueryContextTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoQueryContextTests.java
new file mode 100644
index 00000000000..1f724967820
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoQueryContextTests.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.completion;
+
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.search.suggest.completion.context.GeoQueryContext;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class GeoQueryContextTests extends QueryContextTestCase<GeoQueryContext> {
+
+    public static GeoQueryContext randomGeoQueryContext() {
+        final GeoQueryContext.Builder builder = GeoQueryContext.builder();
+        builder.setGeoPoint(new GeoPoint(randomDouble(), randomDouble()));
+        maybeSet(builder::setBoost, randomIntBetween(1, 10));
+        maybeSet(builder::setPrecision, randomIntBetween(1, 12));
+        List<Integer> neighbours = new ArrayList<>();
+        for (int i = 0; i < randomIntBetween(1, 12); i++) {
+            neighbours.add(randomIntBetween(1, 12));
+        }
+        maybeSet(builder::setNeighbours, neighbours);
+        return builder.build();
+    }
+
+    @Override
+    protected GeoQueryContext createTestModel() {
+        return randomGeoQueryContext();
+    }
+
+    @Override
+    protected GeoQueryContext prototype() {
+        return GeoQueryContext.PROTOTYPE;
+    }
+
+    public void testNullGeoPointIsIllegal() {
+        final GeoQueryContext geoQueryContext = randomGeoQueryContext();
+        final GeoQueryContext.Builder builder = GeoQueryContext.builder()
+            .setNeighbours(geoQueryContext.getNeighbours())
+            .setPrecision(geoQueryContext.getPrecision())
+            .setBoost(geoQueryContext.getBoost());
+        try {
+            builder.build();
+            fail("null geo point is illegal");
+        } catch (NullPointerException e) {
+            assertThat(e.getMessage(), equalTo("geoPoint must not be null"));
+        }
+    }
+
+    public void testIllegalArguments() {
+        final GeoQueryContext.Builder builder = GeoQueryContext.builder();
+
+        try {
+            builder.setGeoPoint(null);
+            fail("geoPoint must not be null");
+        } catch (NullPointerException e) {
+            assertEquals(e.getMessage(), "geoPoint must not be null");
+        }
+        try {
+            builder.setBoost(-randomIntBetween(1, Integer.MAX_VALUE));
+            fail("boost must be positive");
+        } catch (IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "boost must be greater than 0");
+        }
+        int precision = 0;
+        try {
+            do {
+                precision = randomInt();
+            } while (precision >= 1 && precision <= 12);
+            builder.setPrecision(precision);
+        } catch (IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "precision must be between 1 and 12");
+        }
+        try {
+            List<Integer> neighbours = new ArrayList<>();
+            neighbours.add(precision);
+            for (int i = 1; i < randomIntBetween(1, 11); i++) {
+                neighbours.add(i);
+            }
+            Collections.shuffle(neighbours, random());
+            builder.setNeighbours(neighbours);
+        } catch (IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "neighbour value must be between 1 and 12");
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/QueryContextTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/QueryContextTestCase.java
new file mode 100644
index 00000000000..78b73e68890
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/QueryContextTestCase.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.completion;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.suggest.completion.context.QueryContext;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static junit.framework.TestCase.assertEquals;
+
+
+public abstract class QueryContextTestCase<QC extends QueryContext> extends ESTestCase {
+
+    private static final int NUMBER_OF_RUNS = 20;
+
+    /**
+     * create random model that is put under test
+     */
+    protected abstract QC createTestModel();
+
+    /**
+     * query context prototype to read serialized format
+     */
+    protected abstract QC prototype();
+
+    public void testToXContext() throws IOException {
+        for (int i = 0; i < NUMBER_OF_RUNS; i++) {
+            QueryContext toXContent = createTestModel();
+            XContentBuilder builder = XContentFactory.jsonBuilder();
+            toXContent.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            BytesReference bytesReference = builder.bytes();
+            XContentParser parser = XContentFactory.xContent(bytesReference).createParser(bytesReference);
+            parser.nextToken();
+            QueryContext fromXContext = prototype().fromXContext(parser);
+            assertEquals(toXContent, fromXContext);
+            assertEquals(toXContent.hashCode(), fromXContext.hashCode());
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/RegexOptionsTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/RegexOptionsTests.java
new file mode 100644
index 00000000000..082e2bc2687
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/RegexOptionsTests.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.completion;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.index.query.RegexpFlag;
+
+import java.io.IOException;
+
+public class RegexOptionsTests extends WritableTestCase<RegexOptions> {
+
+    public static RegexOptions randomRegexOptions() {
+        final RegexOptions.Builder builder = RegexOptions.builder();
+        maybeSet(builder::setMaxDeterminizedStates, randomIntBetween(1, 1000));
+        StringBuilder sb = new StringBuilder();
+        for (RegexpFlag regexpFlag : RegexpFlag.values()) {
+            if (randomBoolean()) {
+                if (sb.length() != 0) {
+                    sb.append("|");
+                }
+                sb.append(regexpFlag.name());
+            }
+        }
+        maybeSet(builder::setFlags, sb.toString());
+        return builder.build();
+    }
+
+    @Override
+    protected RegexOptions createTestModel() {
+        return randomRegexOptions();
+    }
+
+    @Override
+    protected RegexOptions createMutation(RegexOptions original) throws IOException {
+        final RegexOptions.Builder builder = RegexOptions.builder();
+        builder.setMaxDeterminizedStates(randomValueOtherThan(original.getMaxDeterminizedStates(), () -> randomIntBetween(1, 10)));
+        return builder.build();
+    }
+
+    @Override
+    protected RegexOptions readFrom(StreamInput in) throws IOException {
+        return RegexOptions.readRegexOptions(in);
+    }
+
+    public void testIllegalArgument() {
+        final RegexOptions.Builder builder = RegexOptions.builder();
+        try {
+            builder.setMaxDeterminizedStates(-randomIntBetween(1, Integer.MAX_VALUE));
+            fail("max determinized state must be positive");
+        } catch (IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "maxDeterminizedStates must not be negative");
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/WritableTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/WritableTestCase.java
new file mode 100644
index 00000000000..68cc30f8de4
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/WritableTestCase.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.completion;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+
+/**
+ * Base class for testing serialization and equality for
+ * {@link Writeable} models
+ */
+public abstract class WritableTestCase<M extends Writeable> extends ESTestCase {
+
+    protected static final int NUMBER_OF_RUNS = 20;
+
+    /**
+     * create random model that is put under test
+     */
+    protected abstract M createTestModel();
+
+    /**
+     * mutate the given model so the returned model is different
+     */
+    protected abstract M createMutation(M original) throws IOException;
+
+    /**
+     * model prototype to read serialized format
+     */
+    protected abstract M readFrom(StreamInput in) throws IOException;
+
+    /**
+     * Test serialization and deserialization of the tested model.
+     */
+    public void testSerialization() throws IOException {
+        for (int i = 0; i < NUMBER_OF_RUNS; i++) {
+            M testModel = createTestModel();
+            M deserializedModel = copyModel(testModel);
+            assertEquals(testModel, deserializedModel);
+            assertEquals(testModel.hashCode(), deserializedModel.hashCode());
+            assertNotSame(testModel, deserializedModel);
+        }
+    }
+
+    /**
+     * Test equality and hashCode properties
+     */
+    @SuppressWarnings("unchecked")
+    public void testEqualsAndHashcode() throws IOException {
+        M firstModel = createTestModel();
+        String modelName = firstModel.getClass().getSimpleName();
+        assertFalse(modelName + " is equal to null", firstModel.equals(null));
+        assertFalse(modelName + " is equal to incompatible type", firstModel.equals(""));
+        assertTrue(modelName + " is not equal to self", firstModel.equals(firstModel));
+        assertThat("same "+ modelName + "'s hashcode returns different values if called multiple times", firstModel.hashCode(),
+                equalTo(firstModel.hashCode()));
+        assertThat("different " + modelName + " should not be equal", createMutation(firstModel), not(equalTo(firstModel)));
+
+        M secondModel = copyModel(firstModel);
+        assertTrue(modelName + " is not equal to self", secondModel.equals(secondModel));
+        assertTrue(modelName + " is not equal to its copy", firstModel.equals(secondModel));
+        assertTrue("equals is not symmetric", secondModel.equals(firstModel));
+        assertThat(modelName + " copy's hashcode is different from original hashcode", secondModel.hashCode(),
+                equalTo(firstModel.hashCode()));
+
+        M thirdModel = copyModel(secondModel);
+        assertTrue(modelName + " is not equal to self", thirdModel.equals(thirdModel));
+        assertTrue(modelName + " is not equal to its copy", secondModel.equals(thirdModel));
+        assertThat(modelName + " copy's hashcode is different from original hashcode", secondModel.hashCode(),
+                equalTo(thirdModel.hashCode()));
+        assertTrue("equals is not transitive", firstModel.equals(thirdModel));
+        assertThat(modelName + " copy's hashcode is different from original hashcode", firstModel.hashCode(),
+                equalTo(thirdModel.hashCode()));
+        assertTrue(modelName + " equals is not symmetric", thirdModel.equals(secondModel));
+        assertTrue(modelName + " equals is not symmetric", thirdModel.equals(firstModel));
+    }
+
+    private M copyModel(M original) throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            original.writeTo(output);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), provideNamedWritableRegistry())) {
+                return readFrom(in);
+            }
+        }
+    }
+
+    protected NamedWriteableRegistry provideNamedWritableRegistry() {
+        return new NamedWriteableRegistry();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
index ba3c3492476..4a47be481e0 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
-import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -31,28 +30,14 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisService;
-import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.index.mapper.ContentPath;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.Mapper;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.core.StringFieldMapper;
-import org.elasticsearch.index.mapper.core.StringFieldMapper.StringFieldType;
-import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.IndexSettingsModule;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.function.Consumer;
 
 import static org.hamcrest.Matchers.equalTo;
 
@@ -148,73 +133,22 @@ public class DirectCandidateGeneratorTests extends ESTestCase{
         }
     }
 
-    /**
-     * test that build() outputs a {@link DirectCandidateGenerator} that is similar to the one
-     * we would get when parsing the xContent the test generator is rendering out
-     */
-    public void testBuild() throws IOException {
-
-        long start = System.currentTimeMillis();
-        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), Settings.EMPTY);
-
-        AnalysisService mockAnalysisService = new AnalysisService(idxSettings, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()) {
-            @Override
-            public NamedAnalyzer analyzer(String name) {
-                return new NamedAnalyzer(name, new WhitespaceAnalyzer());
-            }
-        };
-
-        MapperService mockMapperService = new MapperService(idxSettings, mockAnalysisService , null, new IndicesModule().getMapperRegistry(), null) {
-            @Override
-            public MappedFieldType fullName(String fullName) {
-                return new StringFieldType();
-            }
-        };
-
-        QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, mockMapperService, null, null, null) {
-            @Override
-            public MappedFieldType fieldMapper(String name) {
-                TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name);
-                return builder.build(new Mapper.BuilderContext(idxSettings.getSettings(), new ContentPath(1))).fieldType();
-            }
-        };
-        mockShardContext.setMapUnmappedFieldAsString(true);
-
-        for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) {
-            DirectCandidateGeneratorBuilder generator = randomCandidateGenerator();
-            // first, build via DirectCandidateGenerator#build()
-            DirectCandidateGenerator contextGenerator = generator.build(mockShardContext);
-
-            // second, render random test generator to xContent and parse using
-            // PhraseSuggestParser
-            XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
-            if (randomBoolean()) {
-                builder.prettyPrint();
-            }
-            generator.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            XContentParser parser = XContentHelper.createParser(builder.bytes());
-
-            DirectCandidateGenerator secondGenerator = PhraseSuggestParser.parseCandidateGenerator(parser,
-                    mockShardContext.getMapperService(), mockShardContext.parseFieldMatcher());
-
-            // compare their properties
-            assertNotSame(contextGenerator, secondGenerator);
-            assertEquals(contextGenerator.field(), secondGenerator.field());
-            assertEquals(contextGenerator.accuracy(), secondGenerator.accuracy(), Float.MIN_VALUE);
-            assertEquals(contextGenerator.maxTermFreq(), secondGenerator.maxTermFreq(), Float.MIN_VALUE);
-            assertEquals(contextGenerator.maxEdits(), secondGenerator.maxEdits());
-            assertEquals(contextGenerator.maxInspections(), secondGenerator.maxInspections());
-            assertEquals(contextGenerator.minDocFreq(), secondGenerator.minDocFreq(), Float.MIN_VALUE);
-            assertEquals(contextGenerator.minWordLength(), secondGenerator.minWordLength());
-            assertEquals(contextGenerator.postFilter(), secondGenerator.postFilter());
-            assertEquals(contextGenerator.prefixLength(), secondGenerator.prefixLength());
-            assertEquals(contextGenerator.preFilter(), secondGenerator.preFilter());
-            assertEquals(contextGenerator.sort(), secondGenerator.sort());
-            assertEquals(contextGenerator.size(), secondGenerator.size());
-            // some instances of StringDistance don't support equals, just checking the class here
-            assertEquals(contextGenerator.stringDistance().getClass(), secondGenerator.stringDistance().getClass());
-            assertEquals(contextGenerator.suggestMode(), secondGenerator.suggestMode());
-        }
+    public static void assertEqualGenerators(DirectCandidateGenerator first, DirectCandidateGenerator second) {
+        assertEquals(first.field(), second.field());
+        assertEquals(first.accuracy(), second.accuracy(), Float.MIN_VALUE);
+        assertEquals(first.maxTermFreq(), second.maxTermFreq(), Float.MIN_VALUE);
+        assertEquals(first.maxEdits(), second.maxEdits());
+        assertEquals(first.maxInspections(), second.maxInspections());
+        assertEquals(first.minDocFreq(), second.minDocFreq(), Float.MIN_VALUE);
+        assertEquals(first.minWordLength(), second.minWordLength());
+        assertEquals(first.postFilter(), second.postFilter());
+        assertEquals(first.prefixLength(), second.prefixLength());
+        assertEquals(first.preFilter(), second.preFilter());
+        assertEquals(first.sort(), second.sort());
+        assertEquals(first.size(), second.size());
+        // some instances of StringDistance don't support equals, just checking the class here
+        assertEquals(first.stringDistance().getClass(), second.stringDistance().getClass());
+        assertEquals(first.suggestMode(), second.suggestMode());
     }
 
     /**
@@ -306,12 +240,6 @@ public class DirectCandidateGeneratorTests extends ESTestCase{
         return generator;
     }
 
-    private static <T> void maybeSet(Consumer<T> consumer, T value) {
-         if (randomBoolean()) {
-             consumer.accept(value);
-         }
-    }
-
     private static DirectCandidateGeneratorBuilder serializedCopy(DirectCandidateGeneratorBuilder original) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             original.writeTo(output);
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java
index 87ad654e0cd..96ac0c9cb27 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java
@@ -19,15 +19,17 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
-
 import static org.hamcrest.Matchers.instanceOf;
 
 public class LaplaceModelTests extends SmoothingModelTestCase {
 
     @Override
     protected SmoothingModel createTestModel() {
+        return createRandomModel();
+    }
+
+
+    static SmoothingModel createRandomModel() {
         return new Laplace(randomDoubleBetween(0.0, 10.0, false));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java
index 1112b7a5ed7..ed663ef5241 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java
@@ -19,15 +19,16 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
-
 import static org.hamcrest.Matchers.instanceOf;
 
 public class LinearInterpolationModelTests extends SmoothingModelTestCase {
 
     @Override
     protected SmoothingModel createTestModel() {
+        return createRandomModel();
+    }
+
+    static LinearInterpolation createRandomModel() {
         double trigramLambda = randomDoubleBetween(0.0, 10.0, false);
         double bigramLambda = randomDoubleBetween(0.0, 10.0, false);
         double unigramLambda = randomDoubleBetween(0.0, 10.0, false);
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java
new file mode 100644
index 00000000000..36131c80483
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.phrase;
+
+import org.elasticsearch.script.Template;
+import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.instanceOf;
+
+public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestCase<PhraseSuggestionBuilder> {
+
+    @BeforeClass
+    public static void initSmoothingModels() {
+        namedWriteableRegistry.registerPrototype(SmoothingModel.class, Laplace.PROTOTYPE);
+        namedWriteableRegistry.registerPrototype(SmoothingModel.class, LinearInterpolation.PROTOTYPE);
+        namedWriteableRegistry.registerPrototype(SmoothingModel.class, StupidBackoff.PROTOTYPE);
+    }
+
+    @Override
+    protected PhraseSuggestionBuilder randomSuggestionBuilder() {
+        return randomPhraseSuggestionBuilder();
+    }
+
+    public static PhraseSuggestionBuilder randomPhraseSuggestionBuilder() {
+        PhraseSuggestionBuilder testBuilder = new PhraseSuggestionBuilder(randomAsciiOfLengthBetween(2, 20));
+        setCommonPropertiesOnRandomBuilder(testBuilder);
+        maybeSet(testBuilder::maxErrors, randomFloat());
+        maybeSet(testBuilder::separator, randomAsciiOfLengthBetween(1, 10));
+        maybeSet(testBuilder::realWordErrorLikelihood, randomFloat());
+        maybeSet(testBuilder::confidence, randomFloat());
+        maybeSet(testBuilder::collateQuery, randomAsciiOfLengthBetween(3, 20));
+        // collate query prune and parameters will only be used when query is set
+        if (testBuilder.collateQuery() != null) {
+            maybeSet(testBuilder::collatePrune, randomBoolean());
+            if (randomBoolean()) {
+                Map<String, Object> collateParams = new HashMap<>();
+                int numParams = randomIntBetween(1, 5);
+                for (int i = 0; i < numParams; i++) {
+                    collateParams.put(randomAsciiOfLength(5), randomAsciiOfLength(5));
+                }
+                testBuilder.collateParams(collateParams );
+            }
+        }
+        if (randomBoolean()) {
+            // preTag, postTag
+            testBuilder.highlight(randomAsciiOfLengthBetween(3, 20), randomAsciiOfLengthBetween(3, 20));
+        }
+        maybeSet(testBuilder::gramSize, randomIntBetween(1, 5));
+        maybeSet(testBuilder::forceUnigrams, randomBoolean());
+        maybeSet(testBuilder::tokenLimit, randomIntBetween(1, 20));
+        if (randomBoolean()) {
+            testBuilder.smoothingModel(randomSmoothingModel());
+        }
+        if (randomBoolean()) {
+            int numGenerators = randomIntBetween(1, 5);
+            for (int i = 0; i < numGenerators; i++) {
+                testBuilder.addCandidateGenerator(DirectCandidateGeneratorTests.randomCandidateGenerator());
+            }
+        }
+        return testBuilder;
+    }
+
+    private static SmoothingModel randomSmoothingModel() {
+        SmoothingModel model = null;
+        switch (randomIntBetween(0,2)) {
+        case 0:
+            model = LaplaceModelTests.createRandomModel();
+            break;
+        case 1:
+            model = StupidBackoffModelTests.createRandomModel();
+            break;
+        case 2:
+            model = LinearInterpolationModelTests.createRandomModel();
+            break;
+        }
+        return model;
+    }
+
+    @Override
+    protected void mutateSpecificParameters(PhraseSuggestionBuilder builder) throws IOException {
+        switch (randomIntBetween(0, 12)) {
+        case 0:
+            builder.maxErrors(randomValueOtherThan(builder.maxErrors(), () -> randomFloat()));
+            break;
+        case 1:
+            builder.realWordErrorLikelihood(randomValueOtherThan(builder.realWordErrorLikelihood(), () -> randomFloat()));
+            break;
+        case 2:
+            builder.confidence(randomValueOtherThan(builder.confidence(), () -> randomFloat()));
+            break;
+        case 3:
+            builder.gramSize(randomValueOtherThan(builder.gramSize(), () -> randomIntBetween(1, 5)));
+            break;
+        case 4:
+            builder.tokenLimit(randomValueOtherThan(builder.tokenLimit(), () -> randomIntBetween(1, 20)));
+            break;
+        case 5:
+            builder.separator(randomValueOtherThan(builder.separator(), () -> randomAsciiOfLengthBetween(1, 10)));
+            break;
+        case 6:
+            Template collateQuery = builder.collateQuery();
+            if (collateQuery != null) {
+                builder.collateQuery(randomValueOtherThan(collateQuery.getScript(), () -> randomAsciiOfLengthBetween(3, 20)));
+            } else {
+                builder.collateQuery(randomAsciiOfLengthBetween(3, 20));
+            }
+            break;
+        case 7:
+            builder.collatePrune(builder.collatePrune() == null ? randomBoolean() : !builder.collatePrune() );
+            break;
+        case 8:
+            // preTag, postTag
+            String currentPre = builder.preTag();
+            if (currentPre != null) {
+                // simply double both values
+                builder.highlight(builder.preTag() + builder.preTag(), builder.postTag() + builder.postTag());
+            } else {
+                builder.highlight(randomAsciiOfLengthBetween(3, 20), randomAsciiOfLengthBetween(3, 20));
+            }
+            break;
+        case 9:
+            builder.forceUnigrams(builder.forceUnigrams() == null ? randomBoolean() : ! builder.forceUnigrams());
+            break;
+        case 10:
+            Map<String, Object> collateParams = builder.collateParams() == null ? new HashMap<>(1) : builder.collateParams();
+            collateParams.put(randomAsciiOfLength(5), randomAsciiOfLength(5));
+            builder.collateParams(collateParams);
+            break;
+        case 11:
+            builder.smoothingModel(randomValueOtherThan(builder.smoothingModel(), PhraseSuggestionBuilderTests::randomSmoothingModel));
+            break;
+        case 12:
+            builder.addCandidateGenerator(DirectCandidateGeneratorTests.randomCandidateGenerator());
+            break;
+        }
+    }
+
+    public void testInvalidParameters() throws IOException {
+        // test missing field name
+        try {
+            new PhraseSuggestionBuilder(null);
+            fail("Should not allow null as field name");
+        } catch (NullPointerException e) {
+            assertEquals("suggestion requires a field name", e.getMessage());
+        }
+
+        // test emtpy field name
+        try {
+            new PhraseSuggestionBuilder("");
+            fail("Should not allow empty string as field name");
+        } catch (IllegalArgumentException e) {
+            assertEquals("suggestion field name is empty", e.getMessage());
+        }
+
+        PhraseSuggestionBuilder builder = new PhraseSuggestionBuilder(randomAsciiOfLengthBetween(2, 20));
+        try {
+            builder.gramSize(0);
+            fail("Should not allow gramSize < 1");
+        } catch (IllegalArgumentException e) {
+            assertEquals("gramSize must be >= 1", e.getMessage());
+        }
+
+        try {
+            builder.gramSize(-1);
+            fail("Should not allow gramSize < 1");
+        } catch (IllegalArgumentException e) {
+            assertEquals("gramSize must be >= 1", e.getMessage());
+        }
+
+        try {
+            builder.maxErrors(-1);
+            fail("Should not allow maxErrors < 0");
+        } catch (IllegalArgumentException e) {
+            assertEquals("max_error must be > 0.0", e.getMessage());
+        }
+
+        try {
+            builder.separator(null);
+            fail("Should not allow null as separator");
+        } catch (NullPointerException e) {
+            assertEquals("separator cannot be set to null", e.getMessage());
+        }
+
+        try {
+            builder.realWordErrorLikelihood(-1);
+            fail("Should not allow real world error likelihood < 0");
+        } catch (IllegalArgumentException e) {
+            assertEquals("real_word_error_likelihood must be > 0.0", e.getMessage());
+        }
+
+        try {
+            builder.confidence(-1);
+            fail("Should not allow confidence < 0");
+        } catch (IllegalArgumentException e) {
+            assertEquals("confidence must be >= 0.0", e.getMessage());
+        }
+
+        try {
+            builder.tokenLimit(0);
+            fail("token_limit must be >= 1");
+        } catch (IllegalArgumentException e) {
+            assertEquals("token_limit must be >= 1", e.getMessage());
+        }
+
+        try {
+            if (randomBoolean()) {
+                builder.highlight(null, "</b>");
+            } else {
+                builder.highlight("<b>", null);
+            }
+            fail("Pre and post tag must both be null or both not be null.");
+        } catch (IllegalArgumentException e) {
+            assertEquals("Pre and post tag must both be null or both not be null.", e.getMessage());
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
index 51152733bf8..c3890f0eade 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
@@ -45,10 +45,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -97,7 +93,8 @@ public abstract class SmoothingModelTestCase extends ESTestCase {
      * Test that creates new smoothing model from a random test smoothing model and checks both for equality
      */
     public void testFromXContent() throws IOException {
-        QueryParseContext context = new QueryParseContext(new IndicesQueriesRegistry(Settings.settingsBuilder().build(), Collections.emptyMap()));
+        QueryParseContext context = new QueryParseContext(
+                new IndicesQueriesRegistry(Settings.settingsBuilder().build(), Collections.emptyMap()));
         context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY));
 
         SmoothingModel testModel = createTestModel();
@@ -111,9 +108,7 @@ public abstract class SmoothingModelTestCase extends ESTestCase {
         XContentParser parser = XContentHelper.createParser(contentBuilder.bytes());
         context.reset(parser);
         parser.nextToken();  // go to start token, real parsing would do that in the outer element parser
-        SmoothingModel prototype = (SmoothingModel) namedWriteableRegistry.getPrototype(SmoothingModel.class,
-                testModel.getWriteableName());
-        SmoothingModel parsedModel = prototype.fromXContent(context);
+        SmoothingModel parsedModel = testModel.innerFromXContent(context);
         assertNotSame(testModel, parsedModel);
         assertEquals(testModel, parsedModel);
         assertEquals(testModel.hashCode(), parsedModel.hashCode());
@@ -134,7 +129,8 @@ public abstract class SmoothingModelTestCase extends ESTestCase {
         writer.addDocument(doc);
         DirectoryReader ir = DirectoryReader.open(writer);
 
-        WordScorer wordScorer = testModel.buildWordScorerFactory().newScorer(ir, MultiFields.getTerms(ir , "field"), "field", 0.9d, BytesRefs.toBytesRef(" "));
+        WordScorer wordScorer = testModel.buildWordScorerFactory().newScorer(ir, MultiFields.getTerms(ir, "field"), "field", 0.9d,
+                BytesRefs.toBytesRef(" "));
         assertWordScorer(wordScorer, testModel);
     }
 
@@ -159,36 +155,38 @@ public abstract class SmoothingModelTestCase extends ESTestCase {
      */
     @SuppressWarnings("unchecked")
     public void testEqualsAndHashcode() throws IOException {
-            SmoothingModel firstModel = createTestModel();
-            assertFalse("smoothing model is equal to null", firstModel.equals(null));
-            assertFalse("smoothing model is equal to incompatible type", firstModel.equals(""));
-            assertTrue("smoothing model is not equal to self", firstModel.equals(firstModel));
-            assertThat("same smoothing model's hashcode returns different values if called multiple times", firstModel.hashCode(),
-                    equalTo(firstModel.hashCode()));
-            assertThat("different smoothing models should not be equal", createMutation(firstModel), not(equalTo(firstModel)));
+        SmoothingModel firstModel = createTestModel();
+        assertFalse("smoothing model is equal to null", firstModel.equals(null));
+        assertFalse("smoothing model is equal to incompatible type", firstModel.equals(""));
+        assertTrue("smoothing model is not equal to self", firstModel.equals(firstModel));
+        assertThat("same smoothing model's hashcode returns different values if called multiple times", firstModel.hashCode(),
+                equalTo(firstModel.hashCode()));
+        assertThat("different smoothing models should not be equal", createMutation(firstModel), not(equalTo(firstModel)));
 
-            SmoothingModel secondModel = copyModel(firstModel);
-            assertTrue("smoothing model is not equal to self", secondModel.equals(secondModel));
-            assertTrue("smoothing model is not equal to its copy", firstModel.equals(secondModel));
-            assertTrue("equals is not symmetric", secondModel.equals(firstModel));
-            assertThat("smoothing model copy's hashcode is different from original hashcode", secondModel.hashCode(), equalTo(firstModel.hashCode()));
+        SmoothingModel secondModel = copyModel(firstModel);
+        assertTrue("smoothing model is not equal to self", secondModel.equals(secondModel));
+        assertTrue("smoothing model is not equal to its copy", firstModel.equals(secondModel));
+        assertTrue("equals is not symmetric", secondModel.equals(firstModel));
+        assertThat("smoothing model copy's hashcode is different from original hashcode", secondModel.hashCode(),
+                equalTo(firstModel.hashCode()));
 
-            SmoothingModel thirdModel = copyModel(secondModel);
-            assertTrue("smoothing model is not equal to self", thirdModel.equals(thirdModel));
-            assertTrue("smoothing model is not equal to its copy", secondModel.equals(thirdModel));
-            assertThat("smoothing model copy's hashcode is different from original hashcode", secondModel.hashCode(), equalTo(thirdModel.hashCode()));
-            assertTrue("equals is not transitive", firstModel.equals(thirdModel));
-            assertThat("smoothing model copy's hashcode is different from original hashcode", firstModel.hashCode(), equalTo(thirdModel.hashCode()));
-            assertTrue("equals is not symmetric", thirdModel.equals(secondModel));
-            assertTrue("equals is not symmetric", thirdModel.equals(firstModel));
+        SmoothingModel thirdModel = copyModel(secondModel);
+        assertTrue("smoothing model is not equal to self", thirdModel.equals(thirdModel));
+        assertTrue("smoothing model is not equal to its copy", secondModel.equals(thirdModel));
+        assertThat("smoothing model copy's hashcode is different from original hashcode", secondModel.hashCode(),
+                equalTo(thirdModel.hashCode()));
+        assertTrue("equals is not transitive", firstModel.equals(thirdModel));
+        assertThat("smoothing model copy's hashcode is different from original hashcode", firstModel.hashCode(),
+                equalTo(thirdModel.hashCode()));
+        assertTrue("equals is not symmetric", thirdModel.equals(secondModel));
+        assertTrue("equals is not symmetric", thirdModel.equals(firstModel));
     }
 
     static SmoothingModel copyModel(SmoothingModel original) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             original.writeTo(output);
             try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
-                SmoothingModel prototype = (SmoothingModel) namedWriteableRegistry.getPrototype(SmoothingModel.class, original.getWriteableName());
-                return prototype.readFrom(in);
+                return namedWriteableRegistry.getReader(SmoothingModel.class, original.getWriteableName()).read(in);
             }
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java
index c3bd66d2a81..1b6e1cf2c88 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java
@@ -19,15 +19,16 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
-
 import static org.hamcrest.Matchers.instanceOf;
 
 public class StupidBackoffModelTests extends SmoothingModelTestCase {
 
     @Override
     protected SmoothingModel createTestModel() {
+        return createRandomModel();
+    }
+
+    static SmoothingModel createRandomModel() {
         return new StupidBackoff(randomDoubleBetween(0.0, 10.0, false));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/term/SortByTests.java b/core/src/test/java/org/elasticsearch/search/suggest/term/SortByTests.java
new file mode 100644
index 00000000000..870a96dfe73
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/term/SortByTests.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.term;
+
+import org.elasticsearch.common.io.stream.AbstractWriteableEnumTestCase;
+import org.elasticsearch.search.suggest.SortBy;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * Test the {@link SortBy} enum.
+ */
+public class SortByTests extends AbstractWriteableEnumTestCase {
+
+    @Override
+    public void testValidOrdinals() {
+        assertThat(SortBy.SCORE.ordinal(), equalTo(0));
+        assertThat(SortBy.FREQUENCY.ordinal(), equalTo(1));
+    }
+
+    @Override
+    public void testFromString() {
+        assertThat(SortBy.resolve("score"), equalTo(SortBy.SCORE));
+        assertThat(SortBy.resolve("frequency"), equalTo(SortBy.FREQUENCY));
+        final String doesntExist = "doesnt_exist";
+        try {
+            SortBy.resolve(doesntExist);
+            fail("SortBy should not have an element " + doesntExist);
+        } catch (IllegalArgumentException e) {
+        }
+        try {
+            SortBy.resolve(null);
+            fail("SortBy.resolve on a null value should throw an exception.");
+        } catch (NullPointerException e) {
+            assertThat(e.getMessage(), equalTo("Input string is null"));
+        }
+    }
+
+    @Override
+    public void testWriteTo() throws IOException {
+        assertWriteToStream(SortBy.SCORE, 0);
+        assertWriteToStream(SortBy.FREQUENCY, 1);
+    }
+
+    @Override
+    public void testReadFrom() throws IOException {
+        assertReadFromStream(0, SortBy.SCORE);
+        assertReadFromStream(1, SortBy.FREQUENCY);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/term/StringDistanceImplTests.java b/core/src/test/java/org/elasticsearch/search/suggest/term/StringDistanceImplTests.java
new file mode 100644
index 00000000000..4917b2860fb
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/term/StringDistanceImplTests.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.term;
+
+import org.elasticsearch.common.io.stream.AbstractWriteableEnumTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.search.suggest.term.TermSuggestionBuilder.StringDistanceImpl;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * Test for the {@link StringDistanceImpl} enum.
+ */
+public class StringDistanceImplTests extends AbstractWriteableEnumTestCase {
+
+    @Override
+    public void testValidOrdinals() {
+        assertThat(StringDistanceImpl.INTERNAL.ordinal(), equalTo(0));
+        assertThat(StringDistanceImpl.DAMERAU_LEVENSHTEIN.ordinal(), equalTo(1));
+        assertThat(StringDistanceImpl.LEVENSTEIN.ordinal(), equalTo(2));
+        assertThat(StringDistanceImpl.JAROWINKLER.ordinal(), equalTo(3));
+        assertThat(StringDistanceImpl.NGRAM.ordinal(), equalTo(4));
+    }
+
+    @Override
+    public void testFromString() {
+        assertThat(StringDistanceImpl.resolve("internal"), equalTo(StringDistanceImpl.INTERNAL));
+        assertThat(StringDistanceImpl.resolve("damerau_levenshtein"), equalTo(StringDistanceImpl.DAMERAU_LEVENSHTEIN));
+        assertThat(StringDistanceImpl.resolve("levenstein"), equalTo(StringDistanceImpl.LEVENSTEIN));
+        assertThat(StringDistanceImpl.resolve("jarowinkler"), equalTo(StringDistanceImpl.JAROWINKLER));
+        assertThat(StringDistanceImpl.resolve("ngram"), equalTo(StringDistanceImpl.NGRAM));
+        final String doesntExist = "doesnt_exist";
+        try {
+            StringDistanceImpl.resolve(doesntExist);
+            fail("StringDistanceImpl should not have an element " + doesntExist);
+        } catch (IllegalArgumentException e) {
+        }
+        try {
+            StringDistanceImpl.resolve(null);
+            fail("StringDistanceImpl.resolve on a null value should throw an exception.");
+        } catch (NullPointerException e) {
+            assertThat(e.getMessage(), equalTo("Input string is null"));
+        }
+    }
+
+    @Override
+    public void testWriteTo() throws IOException {
+        assertWriteToStream(StringDistanceImpl.INTERNAL, 0);
+        assertWriteToStream(StringDistanceImpl.DAMERAU_LEVENSHTEIN, 1);
+        assertWriteToStream(StringDistanceImpl.LEVENSTEIN, 2);
+        assertWriteToStream(StringDistanceImpl.JAROWINKLER, 3);
+        assertWriteToStream(StringDistanceImpl.NGRAM, 4);
+    }
+
+    @Override
+    public void testReadFrom() throws IOException {
+        assertReadFromStream(0, StringDistanceImpl.INTERNAL);
+        assertReadFromStream(1, StringDistanceImpl.DAMERAU_LEVENSHTEIN);
+        assertReadFromStream(2, StringDistanceImpl.LEVENSTEIN);
+        assertReadFromStream(3, StringDistanceImpl.JAROWINKLER);
+        assertReadFromStream(4, StringDistanceImpl.NGRAM);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/term/SuggestModeTests.java b/core/src/test/java/org/elasticsearch/search/suggest/term/SuggestModeTests.java
new file mode 100644
index 00000000000..ca2274ea4aa
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/term/SuggestModeTests.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.term;
+
+import org.elasticsearch.common.io.stream.AbstractWriteableEnumTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * Test the {@link SuggestMode} enum.
+ */
+public class SuggestModeTests extends AbstractWriteableEnumTestCase {
+
+    @Override
+    public void testValidOrdinals() {
+        assertThat(SuggestMode.MISSING.ordinal(), equalTo(0));
+        assertThat(SuggestMode.POPULAR.ordinal(), equalTo(1));
+        assertThat(SuggestMode.ALWAYS.ordinal(), equalTo(2));
+    }
+
+    @Override
+    public void testFromString() {
+        assertThat(SuggestMode.resolve("missing"), equalTo(SuggestMode.MISSING));
+        assertThat(SuggestMode.resolve("popular"), equalTo(SuggestMode.POPULAR));
+        assertThat(SuggestMode.resolve("always"), equalTo(SuggestMode.ALWAYS));
+        final String doesntExist = "doesnt_exist";
+        try {
+            SuggestMode.resolve(doesntExist);
+            fail("SuggestMode should not have an element " + doesntExist);
+        } catch (IllegalArgumentException e) {
+        }
+        try {
+            SuggestMode.resolve(null);
+            fail("SuggestMode.resolve on a null value should throw an exception.");
+        } catch (NullPointerException e) {
+            assertThat(e.getMessage(), equalTo("Input string is null"));
+        }
+    }
+
+    @Override
+    public void testWriteTo() throws IOException {
+        assertWriteToStream(SuggestMode.MISSING, 0);
+        assertWriteToStream(SuggestMode.POPULAR, 1);
+        assertWriteToStream(SuggestMode.ALWAYS, 2);
+    }
+
+    @Override
+    public void testReadFrom() throws IOException {
+        assertReadFromStream(0, SuggestMode.MISSING);
+        assertReadFromStream(1, SuggestMode.POPULAR);
+        assertReadFromStream(2, SuggestMode.ALWAYS);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java b/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
new file mode 100644
index 00000000000..5e910905d40
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
@@ -0,0 +1,325 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.term;
+
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase;
+import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
+import org.elasticsearch.search.suggest.SortBy;
+import org.elasticsearch.search.suggest.SuggestBuilder;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
+import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.StringDistanceImpl;
+import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
+
+import java.io.IOException;
+import java.util.Locale;
+
+import static org.elasticsearch.search.suggest.DirectSpellcheckerSettings.DEFAULT_ACCURACY;
+import static org.elasticsearch.search.suggest.DirectSpellcheckerSettings.DEFAULT_MAX_EDITS;
+import static org.elasticsearch.search.suggest.DirectSpellcheckerSettings.DEFAULT_MAX_INSPECTIONS;
+import static org.elasticsearch.search.suggest.DirectSpellcheckerSettings.DEFAULT_MAX_TERM_FREQ;
+import static org.elasticsearch.search.suggest.DirectSpellcheckerSettings.DEFAULT_MIN_DOC_FREQ;
+import static org.elasticsearch.search.suggest.DirectSpellcheckerSettings.DEFAULT_MIN_WORD_LENGTH;
+import static org.elasticsearch.search.suggest.DirectSpellcheckerSettings.DEFAULT_PREFIX_LENGTH;
+import static org.hamcrest.Matchers.containsString;
+
+/**
+ * Test the {@link TermSuggestionBuilder} class.
+ */
+public class TermSuggestionBuilderTests extends AbstractSuggestionBuilderTestCase<TermSuggestionBuilder> {
+
+    /**
+     *  creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original
+     */
+    @Override
+    protected TermSuggestionBuilder randomSuggestionBuilder() {
+        return randomTermSuggestionBuilder();
+    }
+
+    /**
+     * Creates a random TermSuggestionBuilder
+     */
+    public static TermSuggestionBuilder randomTermSuggestionBuilder() {
+        TermSuggestionBuilder testBuilder = new TermSuggestionBuilder(randomAsciiOfLengthBetween(2, 20));
+        setCommonPropertiesOnRandomBuilder(testBuilder);
+        maybeSet(testBuilder::suggestMode, randomSuggestMode());
+        maybeSet(testBuilder::accuracy, randomFloat());
+        maybeSet(testBuilder::sort, randomSort());
+        maybeSet(testBuilder::stringDistance, randomStringDistance());
+        maybeSet(testBuilder::maxEdits, randomIntBetween(1, 2));
+        maybeSet(testBuilder::maxInspections, randomInt(Integer.MAX_VALUE));
+        maybeSet(testBuilder::maxTermFreq, randomFloat());
+        maybeSet(testBuilder::prefixLength, randomInt(Integer.MAX_VALUE));
+        maybeSet(testBuilder::minWordLength, randomInt(Integer.MAX_VALUE));
+        maybeSet(testBuilder::minDocFreq, randomFloat());
+        return testBuilder;
+    }
+
+    private static SuggestMode randomSuggestMode() {
+        final int randomVal = randomIntBetween(0, 2);
+        switch (randomVal) {
+            case 0: return SuggestMode.MISSING;
+            case 1: return SuggestMode.POPULAR;
+            case 2: return SuggestMode.ALWAYS;
+            default: throw new IllegalArgumentException("No suggest mode with an ordinal of " + randomVal);
+        }
+    }
+
+    private static SortBy randomSort() {
+        int randomVal = randomIntBetween(0, 1);
+        switch (randomVal) {
+            case 0: return SortBy.SCORE;
+            case 1: return SortBy.FREQUENCY;
+            default: throw new IllegalArgumentException("No sort mode with an ordinal of " + randomVal);
+        }
+    }
+
+    private static StringDistanceImpl randomStringDistance() {
+        int randomVal = randomIntBetween(0, 4);
+        switch (randomVal) {
+            case 0: return StringDistanceImpl.INTERNAL;
+            case 1: return StringDistanceImpl.DAMERAU_LEVENSHTEIN;
+            case 2: return StringDistanceImpl.LEVENSTEIN;
+            case 3: return StringDistanceImpl.JAROWINKLER;
+            case 4: return StringDistanceImpl.NGRAM;
+            default: throw new IllegalArgumentException("No string distance algorithm with an ordinal of " + randomVal);
+        }
+    }
+
+    @Override
+    protected void mutateSpecificParameters(TermSuggestionBuilder builder) throws IOException {
+        switch (randomIntBetween(0, 9)) {
+            case 0:
+                builder.suggestMode(randomValueOtherThan(builder.suggestMode(), () -> randomSuggestMode()));
+                break;
+            case 1:
+                builder.accuracy(randomValueOtherThan(builder.accuracy(), () -> randomFloat()));
+                break;
+            case 2:
+                builder.sort(randomValueOtherThan(builder.sort(), () -> randomSort()));
+                break;
+            case 3:
+                builder.stringDistance(randomValueOtherThan(builder.stringDistance(), () -> randomStringDistance()));
+                break;
+            case 4:
+                builder.maxEdits(randomValueOtherThan(builder.maxEdits(), () -> randomIntBetween(1, 2)));
+                break;
+            case 5:
+                builder.maxInspections(randomValueOtherThan(builder.maxInspections(), () -> randomInt(Integer.MAX_VALUE)));
+                break;
+            case 6:
+                builder.maxTermFreq(randomValueOtherThan(builder.maxTermFreq(), () -> randomFloat()));
+                break;
+            case 7:
+                builder.prefixLength(randomValueOtherThan(builder.prefixLength(), () -> randomInt(Integer.MAX_VALUE)));
+                break;
+            case 8:
+                builder.minWordLength(randomValueOtherThan(builder.minWordLength(), () -> randomInt(Integer.MAX_VALUE)));
+                break;
+            case 9:
+                builder.minDocFreq(randomValueOtherThan(builder.minDocFreq(), () -> randomFloat()));
+                break;
+            default:
+                break; // do nothing
+        }
+    }
+
+    public void testInvalidParameters() throws IOException {
+        // test missing field name
+        try {
+            new TermSuggestionBuilder(null);
+            fail("Should not allow null as field name");
+        } catch (NullPointerException e) {
+            assertEquals("suggestion requires a field name", e.getMessage());
+        }
+
+        // test emtpy field name
+        try {
+            new TermSuggestionBuilder("");
+            fail("Should not allow empty string as field name");
+        } catch (IllegalArgumentException e) {
+            assertEquals("suggestion field name is empty", e.getMessage());
+        }
+
+        TermSuggestionBuilder builder = new TermSuggestionBuilder(randomAsciiOfLengthBetween(2, 20));
+        // test invalid accuracy values
+        try {
+            builder.accuracy(-0.5f);
+            fail("Should not allow accuracy to be set to a negative value.");
+        } catch (IllegalArgumentException e) {
+        }
+        try {
+            builder.accuracy(1.1f);
+            fail("Should not allow accuracy to be greater than 1.0.");
+        } catch (IllegalArgumentException e) {
+        }
+        // test invalid max edit distance values
+        try {
+            builder.maxEdits(0);
+            fail("Should not allow maxEdits to be less than 1.");
+        } catch (IllegalArgumentException e) {
+        }
+        try {
+            builder.maxEdits(-1);
+            fail("Should not allow maxEdits to be a negative value.");
+        } catch (IllegalArgumentException e) {
+        }
+        try {
+            builder.maxEdits(3);
+            fail("Should not allow maxEdits to be greater than 2.");
+        } catch (IllegalArgumentException e) {
+        }
+        // test invalid max inspections values
+        try {
+            builder.maxInspections(-1);
+            fail("Should not allow maxInspections to be a negative value.");
+        } catch (IllegalArgumentException e) {
+        }
+        // test invalid max term freq values
+        try {
+            builder.maxTermFreq(-0.5f);
+            fail("Should not allow max term freq to be a negative value.");
+        } catch (IllegalArgumentException e) {
+        }
+        try {
+            builder.maxTermFreq(1.5f);
+            fail("If max term freq is greater than 1, it must be a whole number.");
+        } catch (IllegalArgumentException e) {
+        }
+        try {
+            builder.maxTermFreq(2.0f); // this should be allowed
+        } catch (IllegalArgumentException e) {
+            fail("A max term freq greater than 1 that is a whole number should be allowed.");
+        }
+        // test invalid min doc freq values
+        try {
+            builder.minDocFreq(-0.5f);
+            fail("Should not allow min doc freq to be a negative value.");
+        } catch (IllegalArgumentException e) {
+        }
+        try {
+            builder.minDocFreq(1.5f);
+            fail("If min doc freq is greater than 1, it must be a whole number.");
+        } catch (IllegalArgumentException e) {
+        }
+        try {
+            builder.minDocFreq(2.0f); // this should be allowed
+        } catch (IllegalArgumentException e) {
+            fail("A min doc freq greater than 1 that is a whole number should be allowed.");
+        }
+        // test invalid min word length values
+        try {
+            builder.minWordLength(0);
+            fail("A min word length < 1 should not be allowed.");
+        } catch (IllegalArgumentException e) {
+        }
+        try {
+            builder.minWordLength(-1);
+            fail("Should not allow min word length to be a negative value.");
+        } catch (IllegalArgumentException e) {
+        }
+        // test invalid prefix length values
+        try {
+            builder.prefixLength(-1);
+            fail("Should not allow prefix length to be a negative value.");
+        } catch (IllegalArgumentException e) {
+        }
+        // test invalid size values
+        try {
+            builder.size(0);
+            fail("Size must be a positive value.");
+        } catch (IllegalArgumentException e) {
+        }
+        try {
+            builder.size(-1);
+            fail("Size must be a positive value.");
+        } catch (IllegalArgumentException e) {
+        }
+        // null values not allowed for enums
+        try {
+            builder.sort(null);
+            fail("Should not allow setting a null sort value.");
+        } catch (NullPointerException e) {
+        }
+        try {
+            builder.stringDistance(null);
+            fail("Should not allow setting a null string distance value.");
+        } catch (NullPointerException e) {
+        }
+        try {
+            builder.suggestMode(null);
+            fail("Should not allow setting a null suggest mode value.");
+        } catch (NullPointerException e) {
+        }
+    }
+
+    public void testDefaultValuesSet() {
+        TermSuggestionBuilder builder = new TermSuggestionBuilder(randomAsciiOfLengthBetween(2, 20));
+        assertEquals(DEFAULT_ACCURACY, builder.accuracy(), Float.MIN_VALUE);
+        assertEquals(DEFAULT_MAX_EDITS, builder.maxEdits());
+        assertEquals(DEFAULT_MAX_INSPECTIONS, builder.maxInspections());
+        assertEquals(DEFAULT_MAX_TERM_FREQ, builder.maxTermFreq(), Float.MIN_VALUE);
+        assertEquals(DEFAULT_MIN_DOC_FREQ, builder.minDocFreq(), Float.MIN_VALUE);
+        assertEquals(DEFAULT_MIN_WORD_LENGTH, builder.minWordLength());
+        assertEquals(DEFAULT_PREFIX_LENGTH, builder.prefixLength());
+        assertEquals(SortBy.SCORE, builder.sort());
+        assertEquals(StringDistanceImpl.INTERNAL, builder.stringDistance());
+        assertEquals(SuggestMode.MISSING, builder.suggestMode());
+    }
+
+    public void testMalformedJson() {
+        final String field = RandomStrings.randomAsciiOfLength(getRandom(), 10).toLowerCase(Locale.ROOT);
+        String suggest = "{\n" +
+                         "  \"bad-payload\" : {\n" +
+                         "    \"text\" : \"the amsterdma meetpu\",\n" +
+                         "    \"term\" : {\n" +
+                         "      \"field\" : { \"" + field + "\" : \"bad-object\" }\n" +
+                         "    }\n" +
+                         "  }\n" +
+                         "}";
+        try {
+            final SuggestBuilder suggestBuilder = SuggestBuilder.fromXContent(newParseContext(suggest), suggesters);
+            fail("Should not have been able to create SuggestBuilder from malformed JSON: " + suggestBuilder);
+        } catch (Exception e) {
+            assertThat(e.getMessage(), containsString("parsing failed"));
+        }
+    }
+
+    private void assertSpellcheckerSettings(DirectSpellcheckerSettings oldSettings, DirectSpellcheckerSettings newSettings) {
+        final double delta = 0.0d;
+        // make sure the objects aren't the same
+        assertNotSame(oldSettings, newSettings);
+        // make sure the objects aren't null
+        assertNotNull(oldSettings);
+        assertNotNull(newSettings);
+        // and now, make sure they are equal..
+        assertEquals(oldSettings.accuracy(), newSettings.accuracy(), delta);
+        assertEquals(oldSettings.maxEdits(), newSettings.maxEdits());
+        assertEquals(oldSettings.maxInspections(), newSettings.maxInspections());
+        assertEquals(oldSettings.maxTermFreq(), newSettings.maxTermFreq(), delta);
+        assertEquals(oldSettings.minDocFreq(), newSettings.minDocFreq(), delta);
+        assertEquals(oldSettings.minWordLength(), newSettings.minWordLength());
+        assertEquals(oldSettings.prefixLength(), newSettings.prefixLength());
+        assertEquals(oldSettings.sort(), newSettings.sort());
+        assertEquals(oldSettings.stringDistance().getClass(), newSettings.stringDistance().getClass());
+        assertEquals(oldSettings.suggestMode().getClass(), newSettings.suggestMode().getClass());
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
index dc803a46412..43056d1fcb1 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
@@ -21,13 +21,13 @@ package org.elasticsearch.snapshots;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.PendingClusterTask;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
diff --git a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
index bd6c2533652..0acc32887ef 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
@@ -31,12 +31,12 @@ import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
@@ -325,7 +325,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
         logger.info("--> execution was blocked on node [{}], shutting it down", blockedNode);
         unblockNode(blockedNode);
 
-        logger.info("--> stopping node", blockedNode);
+        logger.info("--> stopping node [{}]", blockedNode);
         stopNode(blockedNode);
         logger.info("--> waiting for completion");
         SnapshotInfo snapshotInfo = waitForCompletion("test-repo", "test-snap", TimeValue.timeValueSeconds(60));
@@ -379,7 +379,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
         // Make sure that abort makes some progress
         Thread.sleep(100);
         unblockNode(blockedNode);
-        logger.info("--> stopping node", blockedNode);
+        logger.info("--> stopping node [{}]", blockedNode);
         stopNode(blockedNode);
         try {
             DeleteSnapshotResponse deleteSnapshotResponse = deleteSnapshotResponseFuture.actionGet();
@@ -632,8 +632,8 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
         client().admin().cluster().preparePutRepository("test-repo")
                 .setType("mock").setSettings(Settings.settingsBuilder()
                         .put("location", randomRepoPath())
-                        .put("secret.mock.username", "notsecretusername")
-                        .put("secret.mock.password", "verysecretpassword")
+                        .put(MockRepository.Plugin.USERNAME_SETTING.getKey(), "notsecretusername")
+                        .put(MockRepository.Plugin.PASSWORD_SETTING.getKey(), "verysecretpassword")
         ).get();
 
         RestGetRepositoriesAction getRepoAction = internalCluster().getInstance(RestGetRepositoriesAction.class);
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index 9fb2b0f9989..cc52daad5b0 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -40,7 +40,6 @@ import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResp
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.SnapshotsInProgress;
@@ -52,6 +51,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
 import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
@@ -761,7 +761,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         Client client = client();
 
         Path repo = randomRepoPath();
-        logger.info("-->  creating repository at " + repo.toAbsolutePath());
+        logger.info("-->  creating repository at {}", repo.toAbsolutePath());
         assertAcked(client.admin().cluster().preparePutRepository("test-repo")
                 .setType("fs").setSettings(Settings.settingsBuilder()
                         .put("location", repo)
@@ -817,7 +817,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         Client client = client();
 
         Path repo = randomRepoPath();
-        logger.info("-->  creating repository at " + repo.toAbsolutePath());
+        logger.info("-->  creating repository at {}", repo.toAbsolutePath());
         assertAcked(client.admin().cluster().preparePutRepository("test-repo")
                 .setType("fs").setSettings(Settings.settingsBuilder()
                         .put("location", repo)
@@ -855,7 +855,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         Client client = client();
 
         Path repo = randomRepoPath();
-        logger.info("-->  creating repository at " + repo.toAbsolutePath());
+        logger.info("-->  creating repository at {}", repo.toAbsolutePath());
         assertAcked(client.admin().cluster().preparePutRepository("test-repo")
                 .setType("fs").setSettings(Settings.settingsBuilder()
                         .put("location", repo)
@@ -889,7 +889,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         Client client = client();
 
         Path repo = randomRepoPath();
-        logger.info("-->  creating repository at " + repo.toAbsolutePath());
+        logger.info("-->  creating repository at {}", repo.toAbsolutePath());
         assertAcked(client.admin().cluster().preparePutRepository("test-repo")
                 .setType("fs").setSettings(Settings.settingsBuilder()
                         .put("location", repo)
@@ -1448,7 +1448,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
             }
         }
 
-        logger.info("--> checking snapshot status for all currently running and snapshot with empty repository", blockedNode);
+        logger.info("--> checking snapshot status for all currently running and snapshot with empty repository");
         response = client.admin().cluster().prepareSnapshotStatus().execute().actionGet();
         assertThat(response.getSnapshots().size(), equalTo(1));
         snapshotStatus = response.getSnapshots().get(0);
@@ -1461,7 +1461,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
             }
         }
 
-        logger.info("--> checking that _current returns the currently running snapshot", blockedNode);
+        logger.info("--> checking that _current returns the currently running snapshot");
         GetSnapshotsResponse getResponse = client.admin().cluster().prepareGetSnapshots("test-repo").setCurrentSnapshot().execute().actionGet();
         assertThat(getResponse.getSnapshots().size(), equalTo(1));
         SnapshotInfo snapshotInfo = getResponse.getSnapshots().get(0);
@@ -1475,7 +1475,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         logger.info("--> done");
 
 
-        logger.info("--> checking snapshot status again after snapshot is done", blockedNode);
+        logger.info("--> checking snapshot status again after snapshot is done");
         response = client.admin().cluster().prepareSnapshotStatus("test-repo").addSnapshots("test-snap").execute().actionGet();
         snapshotStatus = response.getSnapshots().get(0);
         assertThat(snapshotStatus.getIndices().size(), equalTo(1));
@@ -1486,11 +1486,11 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         assertThat(indexStatus.getShardsStats().getDoneShards(), equalTo(snapshotInfo.successfulShards()));
         assertThat(indexStatus.getShards().size(), equalTo(snapshotInfo.totalShards()));
 
-        logger.info("--> checking snapshot status after it is done with empty repository", blockedNode);
+        logger.info("--> checking snapshot status after it is done with empty repository");
         response = client.admin().cluster().prepareSnapshotStatus().execute().actionGet();
         assertThat(response.getSnapshots().size(), equalTo(0));
 
-        logger.info("--> checking that _current no longer returns the snapshot", blockedNode);
+        logger.info("--> checking that _current no longer returns the snapshot");
         assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("_current").execute().actionGet().getSnapshots().isEmpty(), equalTo(true));
 
         try {
@@ -1813,19 +1813,31 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         }
     }
 
-    public void testDeleteIndexDuringSnapshot() throws Exception {
+    public void testCloseOrDeleteIndexDuringSnapshot() throws Exception {
         Client client = client();
 
         boolean allowPartial = randomBoolean();
-
         logger.info("-->  creating repository");
-        assertAcked(client.admin().cluster().preparePutRepository("test-repo")
+
+        // only block on repo init if we have partial snapshot or we run into deadlock when acquiring shard locks for index deletion/closing
+        boolean initBlocking = allowPartial || randomBoolean();
+        if (initBlocking) {
+            assertAcked(client.admin().cluster().preparePutRepository("test-repo")
                 .setType("mock").setSettings(Settings.settingsBuilder()
-                        .put("location", randomRepoPath())
-                        .put("compress", randomBoolean())
-                        .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
-                        .put("block_on_init", true)
+                    .put("location", randomRepoPath())
+                    .put("compress", randomBoolean())
+                    .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+                    .put("block_on_init", true)
                 ));
+        } else {
+            assertAcked(client.admin().cluster().preparePutRepository("test-repo")
+                .setType("mock").setSettings(Settings.settingsBuilder()
+                    .put("location", randomRepoPath())
+                    .put("compress", randomBoolean())
+                    .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+                    .put("block_on_data", true)
+                ));
+        }
 
         createIndex("test-idx-1", "test-idx-2", "test-idx-3");
         ensureGreen();
@@ -1843,25 +1855,64 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> snapshot allow partial {}", allowPartial);
         ListenableActionFuture<CreateSnapshotResponse> future = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap")
-                .setIndices("test-idx-*").setWaitForCompletion(true).setPartial(allowPartial).execute();
+            .setIndices("test-idx-*").setWaitForCompletion(true).setPartial(allowPartial).execute();
         logger.info("--> wait for block to kick in");
-        waitForBlock(internalCluster().getMasterName(), "test-repo", TimeValue.timeValueMinutes(1));
-        logger.info("--> delete some indices while snapshot is running");
-        client.admin().indices().prepareDelete("test-idx-1", "test-idx-2").get();
-        logger.info("--> unblock running master node");
-        unblockNode(internalCluster().getMasterName());
+        if (initBlocking) {
+            waitForBlock(internalCluster().getMasterName(), "test-repo", TimeValue.timeValueMinutes(1));
+        } else {
+            waitForBlockOnAnyDataNode("test-repo", TimeValue.timeValueMinutes(1));
+        }
+        try {
+            if (allowPartial) {
+                // partial snapshots allow close / delete operations
+                if (randomBoolean()) {
+                    logger.info("--> delete index while partial snapshot is running");
+                    client.admin().indices().prepareDelete("test-idx-1").get();
+                } else {
+                    logger.info("--> close index while partial snapshot is running");
+                    client.admin().indices().prepareClose("test-idx-1").get();
+                }
+            } else {
+                // non-partial snapshots do not allow close / delete operations on indices where snapshot has not been completed
+                if (randomBoolean()) {
+                    try {
+                        logger.info("--> delete index while non-partial snapshot is running");
+                        client.admin().indices().prepareDelete("test-idx-1").get();
+                        fail("Expected deleting index to fail during snapshot");
+                    } catch (IllegalArgumentException e) {
+                        assertThat(e.getMessage(), containsString("Cannot delete indices that are being snapshotted: [[test-idx-1/"));
+                    }
+                } else {
+                    try {
+                        logger.info("--> close index while non-partial snapshot is running");
+                        client.admin().indices().prepareClose("test-idx-1").get();
+                        fail("Expected closing index to fail during snapshot");
+                    } catch (IllegalArgumentException e) {
+                        assertThat(e.getMessage(), containsString("Cannot close indices that are being snapshotted: [[test-idx-1/"));
+                    }
+                }
+            }
+        } finally {
+            if (initBlocking) {
+                logger.info("--> unblock running master node");
+                unblockNode(internalCluster().getMasterName());
+            } else {
+                logger.info("--> unblock all data nodes");
+                unblockAllDataNodes("test-repo");
+            }
+        }
         logger.info("--> waiting for snapshot to finish");
         CreateSnapshotResponse createSnapshotResponse = future.get();
 
         if (allowPartial) {
-            logger.info("Deleted index during snapshot, but allow partial");
+            logger.info("Deleted/Closed index during snapshot, but allow partial");
             assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo((SnapshotState.PARTIAL)));
             assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
             assertThat(createSnapshotResponse.getSnapshotInfo().failedShards(), greaterThan(0));
             assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), lessThan(createSnapshotResponse.getSnapshotInfo().totalShards()));
         } else {
-            logger.info("Deleted index during snapshot and doesn't allow partial");
-            assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo((SnapshotState.FAILED)));
+            logger.info("Snapshot successfully completed");
+            assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo((SnapshotState.SUCCESS)));
         }
     }
 
@@ -1898,24 +1949,28 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         blockAllDataNodes("test-repo");
         logger.info("--> execution will be blocked on all data nodes");
 
-        logger.info("--> start restore");
-        ListenableActionFuture<RestoreSnapshotResponse> restoreFut = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap")
-            .setWaitForCompletion(true)
-            .execute();
-
-        logger.info("--> waiting for block to kick in");
-        waitForBlockOnAnyDataNode("test-repo", TimeValue.timeValueSeconds(60));
-
-        logger.info("--> close index while restore is running");
+        final ListenableActionFuture<RestoreSnapshotResponse> restoreFut;
         try {
-            client.admin().indices().prepareClose("test-idx-1").get();
-            fail("Expected closing index to fail during restore");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("Cannot close indices that are being restored: [test-idx-1]"));
-        }
+            logger.info("--> start restore");
+            restoreFut = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap")
+                .setWaitForCompletion(true)
+                .execute();
 
-        logger.info("--> unblocking all data nodes");
-        unblockAllDataNodes("test-repo");
+            logger.info("--> waiting for block to kick in");
+            waitForBlockOnAnyDataNode("test-repo", TimeValue.timeValueMinutes(1));
+
+            logger.info("--> close index while restore is running");
+            try {
+                client.admin().indices().prepareClose("test-idx-1").get();
+                fail("Expected closing index to fail during restore");
+            } catch (IllegalArgumentException e) {
+                assertThat(e.getMessage(), containsString("Cannot close indices that are being restored: [[test-idx-1/"));
+            }
+        } finally {
+            // unblock even if the try block fails otherwise we will get bogus failures when we delete all indices in test teardown.
+            logger.info("--> unblocking all data nodes");
+            unblockAllDataNodes("test-repo");
+        }
 
         logger.info("--> wait for restore to finish");
         RestoreSnapshotResponse restoreSnapshotResponse = restoreFut.get();
@@ -1960,7 +2015,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
                 shards.put(new ShardId("test-idx", "_na_", 1), new ShardSnapshotStatus("unknown-node", State.ABORTED));
                 shards.put(new ShardId("test-idx", "_na_", 2), new ShardSnapshotStatus("unknown-node", State.ABORTED));
                 List<Entry> entries = new ArrayList<>();
-                entries.add(new Entry(new SnapshotId("test-repo", "test-snap"), true, State.ABORTED, Collections.singletonList("test-idx"), System.currentTimeMillis(), shards.build()));
+                entries.add(new Entry(new SnapshotId("test-repo", "test-snap"), true, false, State.ABORTED, Collections.singletonList("test-idx"), System.currentTimeMillis(), shards.build()));
                 return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(Collections.unmodifiableList(entries))).build();
             }
 
@@ -2111,7 +2166,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
     public void testListCorruptedSnapshot() throws Exception {
         Client client = client();
         Path repo = randomRepoPath();
-        logger.info("-->  creating repository at " + repo.toAbsolutePath());
+        logger.info("-->  creating repository at {}", repo.toAbsolutePath());
         assertAcked(client.admin().cluster().preparePutRepository("test-repo")
                 .setType("fs").setSettings(Settings.settingsBuilder()
                         .put("location", repo)
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotBackwardsCompatibilityIT.java
index eb069d4721c..b3f466cdcc8 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotBackwardsCompatibilityIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotBackwardsCompatibilityIT.java
@@ -215,7 +215,7 @@ public class SnapshotBackwardsCompatibilityIT extends ESBackcompatTestCase {
             logger.info("--> move from 0 to 1 replica");
             client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)).get();
         }
-        logger.debug("---> repo exists: " + Files.exists(tempDir.resolve("indices/test/0")) + " files: " + Arrays.toString(FileSystemUtils.files(tempDir.resolve("indices/test/0")))); // it's only one shard!
+        logger.debug("---> repo exists: {} files: {}", Files.exists(tempDir.resolve("indices/test/0")), Arrays.toString(FileSystemUtils.files(tempDir.resolve("indices/test/0")))); // it's only one shard!
         CreateSnapshotResponse createSnapshotResponseSecond = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-1").setWaitForCompletion(true).setIndices("test").get();
         assertThat(createSnapshotResponseSecond.getSnapshotInfo().successfulShards(), greaterThan(0));
         assertThat(createSnapshotResponseSecond.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponseSecond.getSnapshotInfo().totalShards()));
diff --git a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
index e92a28db86b..426dd2546dd 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
@@ -20,20 +20,19 @@
 package org.elasticsearch.snapshots.mockstore;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.snapshots.IndexShardRepository;
@@ -49,8 +48,6 @@ import java.io.UnsupportedEncodingException;
 import java.nio.file.Path;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
-import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -63,6 +60,10 @@ public class MockRepository extends FsRepository {
 
     public static class Plugin extends org.elasticsearch.plugins.Plugin {
 
+        public static final Setting<String> USERNAME_SETTING = Setting.simpleString("secret.mock.username", Property.NodeScope);
+        public static final Setting<String> PASSWORD_SETTING =
+            Setting.simpleString("secret.mock.password", Property.NodeScope, Property.Filtered);
+
         @Override
         public String name() {
             return "mock-repository";
@@ -78,8 +79,8 @@ public class MockRepository extends FsRepository {
         }
 
         public void onModule(SettingsModule module) {
-            module.registerSettingsFilter("secret.mock.password");
-
+            module.registerSetting(USERNAME_SETTING);
+            module.registerSetting(PASSWORD_SETTING);
         }
     }
 
@@ -120,7 +121,7 @@ public class MockRepository extends FsRepository {
         blockOnInitialization = repositorySettings.settings().getAsBoolean("block_on_init", false);
         randomPrefix = repositorySettings.settings().get("random", "default");
         waitAfterUnblock = repositorySettings.settings().getAsLong("wait_after_unblock", 0L);
-        logger.info("starting mock repository with random prefix " + randomPrefix);
+        logger.info("starting mock repository with random prefix {}", randomPrefix);
         mockBlobStore = new MockBlobStore(super.blobStore());
     }
 
@@ -176,14 +177,12 @@ public class MockRepository extends FsRepository {
     }
 
     public synchronized void unblockExecution() {
-        if (blocked) {
-            blocked = false;
-            // Clean blocking flags, so we wouldn't try to block again
-            blockOnDataFiles = false;
-            blockOnControlFiles = false;
-            blockOnInitialization = false;
-            this.notifyAll();
-        }
+        blocked = false;
+        // Clean blocking flags, so we wouldn't try to block again
+        blockOnDataFiles = false;
+        blockOnControlFiles = false;
+        blockOnInitialization = false;
+        this.notifyAll();
     }
 
     public boolean blocked() {
diff --git a/core/src/test/java/org/elasticsearch/test/MockLogAppender.java b/core/src/test/java/org/elasticsearch/test/MockLogAppender.java
index c0866a81081..9e4a881b25b 100644
--- a/core/src/test/java/org/elasticsearch/test/MockLogAppender.java
+++ b/core/src/test/java/org/elasticsearch/test/MockLogAppender.java
@@ -80,7 +80,7 @@ public class MockLogAppender extends AppenderSkeleton {
         protected final String logger;
         protected final Level level;
         protected final String message;
-        protected boolean saw;
+        volatile boolean saw;
 
         public AbstractEventExpectation(String name, String logger, Level level, String message) {
             this.name = name;
diff --git a/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java b/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java
index 95984da55f6..897fa44b593 100644
--- a/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java
+++ b/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java
@@ -293,8 +293,9 @@ public class RandomShapeGenerator extends RandomGeoGenerator {
                 xDivisible(yRange.getMax()*10e3)/10e3);
     }
 
+    /** creates a small random rectangle by default to keep shape test performance at bay */
     public static Rectangle xRandomRectangle(Random r, Point nearP) {
-        return xRandomRectangle(r, nearP, ctx.getWorldBounds(), false);
+        return xRandomRectangle(r, nearP, ctx.getWorldBounds(), true);
     }
 
     public static Rectangle xRandomRectangle(Random r, Point nearP, boolean small) {
diff --git a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java
index b18be91f575..fc0a8abac27 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java
@@ -65,12 +65,10 @@ public class ThreadPoolStatsTests extends ESTestCase {
         try (BytesStreamOutput os = new BytesStreamOutput()) {
 
             List<ThreadPoolStats.Stats> stats = new ArrayList<>();
-            stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.SUGGEST, -1, 0, 0, 0, 0, 0L));
             stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.SEARCH, -1, 0, 0, 0, 0, 0L));
             stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.WARMER, -1, 0, 0, 0, 0, 0L));
             stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.GENERIC, -1, 0, 0, 0, 0, 0L));
             stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.FORCE_MERGE, -1, 0, 0, 0, 0, 0L));
-            stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.PERCOLATE, -1, 0, 0, 0, 0, 0L));
             stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.SAME, -1, 0, 0, 0, 0, 0L));
 
 
@@ -103,10 +101,8 @@ public class ThreadPoolStatsTests extends ESTestCase {
                 }
                 assertThat(names, contains(ThreadPool.Names.FORCE_MERGE,
                         ThreadPool.Names.GENERIC,
-                        ThreadPool.Names.PERCOLATE,
                         ThreadPool.Names.SAME,
                         ThreadPool.Names.SEARCH,
-                        ThreadPool.Names.SUGGEST,
                         ThreadPool.Names.WARMER));
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index f65ce317d72..2a25b86bc83 100644
--- a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -56,11 +56,11 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
     protected ThreadPool threadPool;
 
-    protected static final Version version0 = Version.fromId(/*0*/99);
+    protected static final Version version0 = Version.CURRENT.minimumCompatibilityVersion();
     protected DiscoveryNode nodeA;
     protected MockTransportService serviceA;
 
-    protected static final Version version1 = Version.fromId(199);
+    protected static final Version version1 = Version.fromId(Version.CURRENT.id+1);
     protected DiscoveryNode nodeB;
     protected MockTransportService serviceB;
 
@@ -137,8 +137,8 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                 try {
                     channel.sendResponse(new StringMessageResponse("hello " + request.message));
                 } catch (IOException e) {
-                    e.printStackTrace();
-                    assertThat(e.getMessage(), false, equalTo(true));
+                    logger.error("Unexpected failure", e);
+                    fail(e.getMessage());
                 }
             }
         });
@@ -162,8 +162,8 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
                     @Override
                     public void handleException(TransportException exp) {
-                        exp.printStackTrace();
-                        assertThat("got exception instead of a response: " + exp.getMessage(), false, equalTo(true));
+                        logger.error("Unexpected failure", exp);
+                        fail("got exception instead of a response: " + exp.getMessage());
                     }
                 });
 
@@ -193,8 +193,8 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
                     @Override
                     public void handleException(TransportException exp) {
-                        exp.printStackTrace();
-                        assertThat("got exception instead of a response: " + exp.getMessage(), false, equalTo(true));
+                        logger.error("Unexpected failure", exp);
+                        fail("got exception instead of a response: " + exp.getMessage());
                     }
                 });
 
@@ -218,7 +218,8 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                 threadPool.getThreadContext().putHeader("test.pong.user", "pong_user");
                 channel.sendResponse(response);
             } catch (IOException e) {
-                assertThat(e.getMessage(), false, equalTo(true));
+                logger.error("Unexpected failure", e);
+                fail(e.getMessage());
             }
         });
         final Object context = new Object();
@@ -245,7 +246,8 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
             @Override
             public void handleException(TransportException exp) {
-                assertThat("got exception instead of a response: " + exp.getMessage(), false, equalTo(true));
+                logger.error("Unexpected failure", exp);
+                fail("got exception instead of a response: " + exp.getMessage());
             }
         };
         StringMessageRequest ping = new StringMessageRequest("ping");
@@ -317,8 +319,8 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                 try {
                     channel.sendResponse(TransportResponse.Empty.INSTANCE, TransportResponseOptions.builder().withCompress(true).build());
                 } catch (IOException e) {
-                    e.printStackTrace();
-                    assertThat(e.getMessage(), false, equalTo(true));
+                    logger.error("Unexpected failure", e);
+                    fail(e.getMessage());
                 }
             }
         });
@@ -341,8 +343,8 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
                     @Override
                     public void handleException(TransportException exp) {
-                        exp.printStackTrace();
-                        assertThat("got exception instead of a response: " + exp.getMessage(), false, equalTo(true));
+                        logger.error("Unexpected failure", exp);
+                        fail("got exception instead of a response: " + exp.getMessage());
                     }
                 });
 
@@ -364,8 +366,8 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                 try {
                     channel.sendResponse(new StringMessageResponse("hello " + request.message), TransportResponseOptions.builder().withCompress(true).build());
                 } catch (IOException e) {
-                    e.printStackTrace();
-                    assertThat(e.getMessage(), false, equalTo(true));
+                    logger.error("Unexpected failure", e);
+                    fail(e.getMessage());
                 }
             }
         });
@@ -389,8 +391,8 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
                     @Override
                     public void handleException(TransportException exp) {
-                        exp.printStackTrace();
-                        assertThat("got exception instead of a response: " + exp.getMessage(), false, equalTo(true));
+                        logger.error("Unexpected failure", exp);
+                        fail("got exception instead of a response: " + exp.getMessage());
                     }
                 });
 
@@ -540,20 +542,21 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
     }
 
     public void testTimeoutSendExceptionWithDelayedResponse() throws Exception {
+        CountDownLatch doneLatch = new CountDownLatch(1);
         serviceA.registerRequestHandler("sayHelloTimeoutDelayedResponse", StringMessageRequest::new, ThreadPool.Names.GENERIC, new TransportRequestHandler<StringMessageRequest>() {
             @Override
             public void messageReceived(StringMessageRequest request, TransportChannel channel) {
                 TimeValue sleep = TimeValue.parseTimeValue(request.message, null, "sleep");
                 try {
-                    Thread.sleep(sleep.millis());
+                    doneLatch.await(sleep.millis(), TimeUnit.MILLISECONDS);
                 } catch (InterruptedException e) {
                     // ignore
                 }
                 try {
                     channel.sendResponse(new StringMessageResponse("hello " + request.message));
                 } catch (IOException e) {
-                    e.printStackTrace();
-                    assertThat(e.getMessage(), false, equalTo(true));
+                    logger.error("Unexpected failure", e);
+                    fail(e.getMessage());
                 }
             }
         });
@@ -613,7 +616,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
                         @Override
                         public void handleException(TransportException exp) {
-                            exp.printStackTrace();
+                            logger.error("Unexpected failure", exp);
                             fail("got exception instead of a response for " + counter + ": " + exp.getDetailedMessage());
                         }
                     });
@@ -623,6 +626,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         }
 
         serviceA.removeHandler("sayHelloTimeoutDelayedResponse");
+        doneLatch.countDown();
     }
 
     @TestLogging(value = "test. transport.tracer:TRACE")
@@ -959,8 +963,8 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
             @Override
             public void handleException(TransportException exp) {
-                exp.printStackTrace();
-                fail();
+                logger.error("Unexpected failure", exp);
+                fail("got exception instead of a response: " + exp.getMessage());
             }
 
             @Override
@@ -1000,8 +1004,8 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
             @Override
             public void handleException(TransportException exp) {
-                exp.printStackTrace();
-                fail();
+                logger.error("Unexpected failure", exp);
+                fail("got exception instead of a response: " + exp.getMessage());
             }
 
             @Override
@@ -1044,8 +1048,8 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
             @Override
             public void handleException(TransportException exp) {
-                exp.printStackTrace();
-                fail();
+                logger.error("Unexpected failure", exp);
+                fail("got exception instead of a response: " + exp.getMessage());
             }
 
             @Override
@@ -1084,8 +1088,8 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
             @Override
             public void handleException(TransportException exp) {
-                exp.printStackTrace();
-                fail();
+                logger.error("Unexpected failure", exp);
+                fail("got exception instead of a response: " + exp.getMessage());
             }
 
             @Override
diff --git a/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java b/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java
index cebd90ec08b..d587ab05e45 100644
--- a/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java
@@ -41,8 +41,8 @@ public class TransportModuleTests extends ModuleTestCase {
 
     static class FakeTransportService extends TransportService {
         @Inject
-        public FakeTransportService(Settings settings, Transport transport, ThreadPool threadPool, NamedWriteableRegistry namedWriteableRegistry) {
-            super(settings, transport, threadPool, namedWriteableRegistry);
+        public FakeTransportService(Settings settings, Transport transport, ThreadPool threadPool) {
+            super(settings, transport, threadPool);
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/KeyedLockTests.java b/core/src/test/java/org/elasticsearch/transport/netty/KeyedLockTests.java
index 9581dfff42f..f9451375590 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/KeyedLockTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/KeyedLockTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.transport.netty;
 
+import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.util.concurrent.KeyedLock;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
@@ -29,9 +30,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 
 public class KeyedLockTests extends ESTestCase {
@@ -68,28 +67,6 @@ public class KeyedLockTests extends ESTestCase {
         }
     }
 
-    public void testCannotAcquireTwoLocks() throws InterruptedException {
-        KeyedLock<String> connectionLock = new KeyedLock<String>();
-        String name = randomRealisticUnicodeOfLength(scaledRandomIntBetween(10, 50));
-        connectionLock.acquire(name);
-        try {
-            connectionLock.acquire(name);
-            fail("Expected IllegalStateException");
-        } catch (IllegalStateException e) {
-            assertThat(e.getMessage(), containsString("Lock already acquired"));
-        }
-    }
-
-    public void testCannotReleaseUnacquiredLock() throws InterruptedException {
-        KeyedLock<String> connectionLock = new KeyedLock<String>();
-        String name = randomRealisticUnicodeOfLength(scaledRandomIntBetween(10, 50));
-        try {
-            connectionLock.release(name);
-            fail("Expected IllegalStateException");
-        } catch (IllegalStateException e) {
-            assertThat(e.getMessage(), is("Lock not acquired"));
-        }
-    }
 
     public static class AcquireAndReleaseThread extends Thread {
         private CountDownLatch startLatch;
@@ -117,16 +94,16 @@ public class KeyedLockTests extends ESTestCase {
             int numRuns = scaledRandomIntBetween(5000, 50000);
             for (int i = 0; i < numRuns; i++) {
                 String curName = names[randomInt(names.length - 1)];
-                connectionLock.acquire(curName);
-                try {
+                assert connectionLock.isHeldByCurrentThread(curName) == false;
+                try (Releasable ignored = connectionLock.acquire(curName)) {
+                    assert connectionLock.isHeldByCurrentThread(curName);
+                    assert connectionLock.isHeldByCurrentThread(curName + "bla") == false;
                     Integer integer = counter.get(curName);
                     if (integer == null) {
                         counter.put(curName, 1);
                     } else {
                         counter.put(curName, integer.intValue() + 1);
                     }
-                } finally {
-                    connectionLock.release(curName);
                 }
                 AtomicInteger atomicInteger = new AtomicInteger(0);
                 AtomicInteger value = safeCounter.putIfAbsent(curName, atomicInteger);
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
index 3d6ff1cd067..f8ffbd360c7 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
@@ -54,13 +54,13 @@ public class NettyScheduledPingTests extends ESTestCase {
 
         NamedWriteableRegistry registryA = new NamedWriteableRegistry();
         final NettyTransport nettyA = new NettyTransport(settings, threadPool, new NetworkService(settings), BigArrays.NON_RECYCLING_INSTANCE, Version.CURRENT, registryA);
-        MockTransportService serviceA = new MockTransportService(settings, nettyA, threadPool, registryA);
+        MockTransportService serviceA = new MockTransportService(settings, nettyA, threadPool);
         serviceA.start();
         serviceA.acceptIncomingRequests();
 
         NamedWriteableRegistry registryB = new NamedWriteableRegistry();
         final NettyTransport nettyB = new NettyTransport(settings, threadPool, new NetworkService(settings), BigArrays.NON_RECYCLING_INSTANCE, Version.CURRENT, registryB);
-        MockTransportService serviceB = new MockTransportService(settings, nettyB, threadPool, registryB);
+        MockTransportService serviceB = new MockTransportService(settings, nettyB, threadPool);
         serviceB.start();
         serviceB.acceptIncomingRequests();
 
@@ -86,8 +86,8 @@ public class NettyScheduledPingTests extends ESTestCase {
                 try {
                     channel.sendResponse(TransportResponse.Empty.INSTANCE, TransportResponseOptions.EMPTY);
                 } catch (IOException e) {
-                    e.printStackTrace();
-                    assertThat(e.getMessage(), false, equalTo(true));
+                    logger.error("Unexpected failure", e);
+                    fail(e.getMessage());
                 }
             }
         });
@@ -113,8 +113,8 @@ public class NettyScheduledPingTests extends ESTestCase {
 
                         @Override
                         public void handleException(TransportException exp) {
-                            exp.printStackTrace();
-                            assertThat("got exception instead of a response: " + exp.getMessage(), false, equalTo(true));
+                            logger.error("Unexpected failure", exp);
+                            fail("got exception instead of a response: " + exp.getMessage());
                         }
                     }).txGet();
         }
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java
index ef408d16784..d9466d28424 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java
@@ -157,7 +157,7 @@ public class NettyTransportIT extends ESIntegTestCase {
                             try {
                                 transportChannel.sendResponse(e);
                             } catch (IOException e1) {
-                                logger.warn("Failed to send error message back to client for action [" + action + "]", e);
+                                logger.warn("Failed to send error message back to client for action [{}]", e, action);
                                 logger.warn("Actual Exception", e1);
                             }
                         }
@@ -194,7 +194,7 @@ public class NettyTransportIT extends ESIntegTestCase {
                                 try {
                                     transportChannel.sendResponse(e);
                                 } catch (Throwable e1) {
-                                    logger.warn("Failed to send error message back to client for action [" + reg.getAction() + "]", e1);
+                                    logger.warn("Failed to send error message back to client for action [{}]", e1, reg.getAction());
                                     logger.warn("Actual Exception", e);
                                 }
                             }                        }
diff --git a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java b/core/src/test/java/org/elasticsearch/tribe/TribeIT.java
index 55a79ffddfc..7313d880a63 100644
--- a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java
+++ b/core/src/test/java/org/elasticsearch/tribe/TribeIT.java
@@ -132,6 +132,9 @@ public class TribeIT extends ESIntegTestCase {
         Settings.Builder tribe1Defaults = Settings.builder();
         Settings.Builder tribe2Defaults = Settings.builder();
         for (Map.Entry<String, String> entry : asMap.entrySet()) {
+            if (entry.getKey().startsWith("path.")) {
+                continue;
+            }
             tribe1Defaults.put("tribe.t1." + entry.getKey(), entry.getValue());
             tribe2Defaults.put("tribe.t2." + entry.getKey(), entry.getValue());
         }
diff --git a/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java b/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java
new file mode 100644
index 00000000000..5174a317a40
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tribe;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+
+public class TribeServiceTests extends ESTestCase {
+    public void testMinimalSettings() {
+        Settings globalSettings = Settings.builder()
+            .put("node.name", "nodename")
+            .put("path.home", "some/path").build();
+        Settings clientSettings = TribeService.buildClientSettings("tribe1", globalSettings, Settings.EMPTY);
+        assertEquals("some/path", clientSettings.get("path.home"));
+        assertEquals("nodename/tribe1", clientSettings.get("node.name"));
+        assertEquals("tribe1", clientSettings.get("tribe.name"));
+        assertEquals("false", clientSettings.get("http.enabled"));
+        assertEquals("true", clientSettings.get("node.client"));
+        assertEquals(5, clientSettings.getAsMap().size());
+    }
+
+    public void testEnvironmentSettings() {
+        Settings globalSettings = Settings.builder()
+            .put("node.name", "nodename")
+            .put("path.home", "some/path")
+            .put("path.conf", "conf/path")
+            .put("path.plugins", "plugins/path")
+            .put("path.scripts", "scripts/path")
+            .put("path.logs", "logs/path").build();
+        Settings clientSettings = TribeService.buildClientSettings("tribe1", globalSettings, Settings.EMPTY);
+        assertEquals("some/path", clientSettings.get("path.home"));
+        assertEquals("conf/path", clientSettings.get("path.conf"));
+        assertEquals("plugins/path", clientSettings.get("path.plugins"));
+        assertEquals("scripts/path", clientSettings.get("path.scripts"));
+        assertEquals("logs/path", clientSettings.get("path.logs"));
+
+        Settings tribeSettings = Settings.builder()
+            .put("path.home", "alternate/path").build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
+            TribeService.buildClientSettings("tribe1", globalSettings, tribeSettings);
+        });
+        assertTrue(e.getMessage(), e.getMessage().contains("Setting [path.home] not allowed in tribe client"));
+    }
+
+    public void testPassthroughSettings() {
+        Settings globalSettings = Settings.builder()
+            .put("node.name", "nodename")
+            .put("path.home", "some/path")
+            .put("network.host", "0.0.0.0")
+            .put("network.bind_host", "1.1.1.1")
+            .put("network.publish_host", "2.2.2.2")
+            .put("transport.host", "3.3.3.3")
+            .put("transport.bind_host", "4.4.4.4")
+            .put("transport.publish_host", "5.5.5.5").build();
+        Settings clientSettings = TribeService.buildClientSettings("tribe1", globalSettings, Settings.EMPTY);
+        assertEquals("0.0.0.0", clientSettings.get("network.host"));
+        assertEquals("1.1.1.1", clientSettings.get("network.bind_host"));
+        assertEquals("2.2.2.2", clientSettings.get("network.publish_host"));
+        assertEquals("3.3.3.3", clientSettings.get("transport.host"));
+        assertEquals("4.4.4.4", clientSettings.get("transport.bind_host"));
+        assertEquals("5.5.5.5", clientSettings.get("transport.publish_host"));
+
+        // per tribe client overrides still work
+        Settings tribeSettings = Settings.builder()
+            .put("network.host", "3.3.3.3")
+            .put("network.bind_host", "4.4.4.4")
+            .put("network.publish_host", "5.5.5.5")
+            .put("transport.host", "6.6.6.6")
+            .put("transport.bind_host", "7.7.7.7")
+            .put("transport.publish_host", "8.8.8.8").build();
+        clientSettings = TribeService.buildClientSettings("tribe1", globalSettings, tribeSettings);
+        assertEquals("3.3.3.3", clientSettings.get("network.host"));
+        assertEquals("4.4.4.4", clientSettings.get("network.bind_host"));
+        assertEquals("5.5.5.5", clientSettings.get("network.publish_host"));
+        assertEquals("6.6.6.6", clientSettings.get("transport.host"));
+        assertEquals("7.7.7.7", clientSettings.get("transport.bind_host"));
+        assertEquals("8.8.8.8", clientSettings.get("transport.publish_host"));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java b/core/src/test/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java
index bb8636d36e9..6b69a038a00 100644
--- a/core/src/test/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java
+++ b/core/src/test/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java
@@ -54,7 +54,7 @@ public class ConcurrentDocumentOperationIT extends ESIntegTestCase {
 
                 @Override
                 public void onFailure(Throwable e) {
-                    e.printStackTrace();
+                    logger.error("Unexpected exception while indexing", e);
                     failure.set(e);
                     latch.countDown();
                 }
diff --git a/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java b/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java
index 3432411b225..4e319c65bac 100644
--- a/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java
+++ b/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java
@@ -354,9 +354,7 @@ public class SimpleVersioningIT extends ESIntegTestCase {
         switch (random.nextInt(6)) {
             case 0:
                 // random simple
-                if (VERBOSE) {
-                    System.out.println("TEST: use random simple ids");
-                }
+                logger.info("--> use random simple ids");
                 ids = new IDSource() {
                     @Override
                     public String next() {
@@ -366,9 +364,7 @@ public class SimpleVersioningIT extends ESIntegTestCase {
                 break;
             case 1:
                 // random realistic unicode
-                if (VERBOSE) {
-                    System.out.println("TEST: use random realistic unicode ids");
-                }
+                logger.info("--> use random realistic unicode ids");
                 ids = new IDSource() {
                     @Override
                     public String next() {
@@ -378,9 +374,7 @@ public class SimpleVersioningIT extends ESIntegTestCase {
                 break;
             case 2:
                 // sequential
-                if (VERBOSE) {
-                    System.out.println("TEST: use seuquential ids");
-                }
+                logger.info("--> use sequential ids");
                 ids = new IDSource() {
                     int upto;
 
@@ -392,9 +386,7 @@ public class SimpleVersioningIT extends ESIntegTestCase {
                 break;
             case 3:
                 // zero-pad sequential
-                if (VERBOSE) {
-                    System.out.println("TEST: use zero-pad seuquential ids");
-                }
+                logger.info("--> use zero-padded sequential ids");
                 ids = new IDSource() {
                     final int radix = TestUtil.nextInt(random, Character.MIN_RADIX, Character.MAX_RADIX);
                     final String zeroPad = String.format(Locale.ROOT, "%0" + TestUtil.nextInt(random, 4, 20) + "d", 0);
@@ -409,9 +401,7 @@ public class SimpleVersioningIT extends ESIntegTestCase {
                 break;
             case 4:
                 // random long
-                if (VERBOSE) {
-                    System.out.println("TEST: use random long ids");
-                }
+                logger.info("--> use random long ids");
                 ids = new IDSource() {
                     final int radix = TestUtil.nextInt(random, Character.MIN_RADIX, Character.MAX_RADIX);
                     int upto;
@@ -424,9 +414,7 @@ public class SimpleVersioningIT extends ESIntegTestCase {
                 break;
             case 5:
                 // zero-pad random long
-                if (VERBOSE) {
-                    System.out.println("TEST: use zero-pad random long ids");
-                }
+                logger.info("--> use zero-padded random long ids");
                 ids = new IDSource() {
                     final int radix = TestUtil.nextInt(random, Character.MIN_RADIX, Character.MAX_RADIX);
                     final String zeroPad = String.format(Locale.ROOT, "%015d", 0);
@@ -539,9 +527,7 @@ public class SimpleVersioningIT extends ESIntegTestCase {
             idPrefix = "";
         } else {
             idPrefix = TestUtil.randomSimpleString(random);
-            if (VERBOSE) {
-                System.out.println("TEST: use id prefix: " + idPrefix);
-            }
+            logger.debug("--> use id prefix {}", idPrefix);
         }
 
         int numIDs;
@@ -564,9 +550,7 @@ public class SimpleVersioningIT extends ESIntegTestCase {
         final IDAndVersion[] idVersions = new IDAndVersion[TestUtil.nextInt(random, numIDs / 2, numIDs * (TEST_NIGHTLY ? 8 : 2))];
         final Map<String, IDAndVersion> truth = new HashMap<>();
 
-        if (VERBOSE) {
-            System.out.println("TEST: use " + numIDs + " ids; " + idVersions.length + " operations");
-        }
+        logger.debug("--> use {} ids; {} operations", numIDs, idVersions.length);
 
         for (int i = 0; i < idVersions.length; i++) {
 
@@ -596,10 +580,9 @@ public class SimpleVersioningIT extends ESIntegTestCase {
             idVersions[i] = x;
         }
 
-        if (VERBOSE) {
-            for (IDAndVersion idVersion : idVersions) {
-                System.out.println("id=" + idVersion.id + " version=" + idVersion.version + " delete?=" + idVersion.delete + " truth?=" + (truth.get(idVersion.id) == idVersion));
-            }
+        for (IDAndVersion idVersion : idVersions) {
+            logger.debug("--> id={} version={} delete?={} truth?={}", idVersion.id, idVersion.version, idVersion.delete,
+                truth.get(idVersion.id) == idVersion);
         }
 
         final AtomicInteger upto = new AtomicInteger();
@@ -623,8 +606,8 @@ public class SimpleVersioningIT extends ESIntegTestCase {
                             if (index >= idVersions.length) {
                                 break;
                             }
-                            if (VERBOSE && index % 100 == 0) {
-                                System.out.println(Thread.currentThread().getName() + ": index=" + index);
+                            if (index % 100 == 0) {
+                                logger.trace("{}: index={}", Thread.currentThread().getName(), index);
                             }
                             IDAndVersion idVersion = idVersions[index];
 
@@ -657,18 +640,18 @@ public class SimpleVersioningIT extends ESIntegTestCase {
                             idVersion.indexFinishTime = System.nanoTime() - startTime;
 
                             if (threadRandom.nextInt(100) == 7) {
-                                System.out.println(threadID + ": TEST: now refresh at " + (System.nanoTime() - startTime));
+                                logger.trace("--> {}: TEST: now refresh at {}", threadID, System.nanoTime() - startTime);
                                 refresh();
-                                System.out.println(threadID + ": TEST: refresh done at " + (System.nanoTime() - startTime));
+                                logger.trace("--> {}: TEST: refresh done at {}", threadID, System.nanoTime() - startTime);
                             }
                             if (threadRandom.nextInt(100) == 7) {
-                                System.out.println(threadID + ": TEST: now flush at " + (System.nanoTime() - startTime));
+                                logger.trace("--> {}: TEST: now flush at {}", threadID, System.nanoTime() - startTime);
                                 try {
                                     flush();
                                 } catch (FlushNotAllowedEngineException fnaee) {
                                     // OK
                                 }
-                                System.out.println(threadID + ": TEST: flush done at " + (System.nanoTime() - startTime));
+                                logger.trace("--> {}: TEST: flush done at {}", threadID, System.nanoTime() - startTime);
                             }
                         }
                     } catch (Exception e) {
@@ -696,16 +679,17 @@ public class SimpleVersioningIT extends ESIntegTestCase {
             }
             long actualVersion = client().prepareGet("test", "type", id).execute().actionGet().getVersion();
             if (actualVersion != expected) {
-                System.out.println("FAILED: idVersion=" + idVersion + " actualVersion=" + actualVersion);
+                logger.error("--> FAILED: idVersion={} actualVersion= {}", idVersion, actualVersion);
                 failed = true;
             }
         }
 
         if (failed) {
-            System.out.println("All versions:");
+            StringBuilder sb = new StringBuilder();
             for (int i = 0; i < idVersions.length; i++) {
-                System.out.println("i=" + i + " " + idVersions[i]);
+                sb.append("i=").append(i).append(" ").append(idVersions[i]).append(System.lineSeparator());
             }
+            logger.error("All versions: {}", sb);
             fail("wrong versions for some IDs");
         }
     }
diff --git a/core/src/test/resources/indices/bwc/index-2.2.0.zip b/core/src/test/resources/indices/bwc/index-2.2.0.zip
index b645084eeef..797ca24f4ed 100644
Binary files a/core/src/test/resources/indices/bwc/index-2.2.0.zip and b/core/src/test/resources/indices/bwc/index-2.2.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-2.2.1.zip b/core/src/test/resources/indices/bwc/index-2.2.1.zip
new file mode 100644
index 00000000000..8d8e55ae62d
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-2.2.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.2.0.zip b/core/src/test/resources/indices/bwc/repo-2.2.0.zip
index f895e11fdd7..f2208b734c0 100644
Binary files a/core/src/test/resources/indices/bwc/repo-2.2.0.zip and b/core/src/test/resources/indices/bwc/repo-2.2.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-2.2.1.zip b/core/src/test/resources/indices/bwc/repo-2.2.1.zip
new file mode 100644
index 00000000000..35f7425b20e
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-2.2.1.zip differ
diff --git a/core/src/test/resources/org/elasticsearch/common/logging/config/logging.yml b/core/src/test/resources/org/elasticsearch/common/logging/config/logging.yml
index bd7a15f4434..515e4320fd2 100644
--- a/core/src/test/resources/org/elasticsearch/common/logging/config/logging.yml
+++ b/core/src/test/resources/org/elasticsearch/common/logging/config/logging.yml
@@ -1,4 +1,4 @@
-# you can override this using by setting a system property, for example -Des.logger.level=DEBUG
+# you can override this using by setting a system property, for example -Ees.logger.level=DEBUG
 es.logger.level: INFO
 rootLogger: ${es.logger.level}, console
 logger:
diff --git a/core/src/test/resources/org/elasticsearch/index/mapper/all/mapping.json b/core/src/test/resources/org/elasticsearch/index/mapper/all/mapping.json
index e0a7bfeba89..80fb091b973 100644
--- a/core/src/test/resources/org/elasticsearch/index/mapper/all/mapping.json
+++ b/core/src/test/resources/org/elasticsearch/index/mapper/all/mapping.json
@@ -2,7 +2,7 @@
     "person":{
         "_all":{
             "enabled":true,
-            "omit_norms":true
+            "norms":false
         },
         "properties":{
             "name":{
diff --git a/core/src/test/resources/org/elasticsearch/index/mapper/multifield/test-multi-fields.json b/core/src/test/resources/org/elasticsearch/index/mapper/multifield/test-multi-fields.json
index 528cd5bb9d2..7d4f819a250 100644
--- a/core/src/test/resources/org/elasticsearch/index/mapper/multifield/test-multi-fields.json
+++ b/core/src/test/resources/org/elasticsearch/index/mapper/multifield/test-multi-fields.json
@@ -17,9 +17,7 @@
           "test1": {
             "type": "text",
             "store": true,
-            "fielddata": {
-              "loading": "eager"
-            }
+            "eager_global_ordinals": true
           },
           "test2": {
             "type": "token_count",
diff --git a/core/src/test/resources/org/elasticsearch/index/mapper/update/all_mapping_create_index.json b/core/src/test/resources/org/elasticsearch/index/mapper/update/all_mapping_create_index.json
index e9604ae458f..02354ecccdd 100644
--- a/core/src/test/resources/org/elasticsearch/index/mapper/update/all_mapping_create_index.json
+++ b/core/src/test/resources/org/elasticsearch/index/mapper/update/all_mapping_create_index.json
@@ -7,13 +7,10 @@
         "store_term_vector_offsets": true,
         "store_term_vector_positions": true,
         "store_term_vector_payloads": true,
-        "omit_norms": true,
+        "norms": false,
         "analyzer": "standard",
         "search_analyzer": "whitespace",
-        "similarity": "my_similarity",
-        "fielddata": {
-           "format": "paged_bytes"
-        }
+        "similarity": "my_similarity"
       }
     }
   },
diff --git a/core/src/test/resources/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json b/core/src/test/resources/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json
index 6ddde341fc2..f9ab776ce5a 100644
--- a/core/src/test/resources/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json
+++ b/core/src/test/resources/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json
@@ -7,13 +7,10 @@
       "store_term_vector_offsets": false,
       "store_term_vector_positions": false,
       "store_term_vector_payloads": false,
-      "omit_norms": false,
+      "norms": true,
       "analyzer": "whitespace",
       "search_analyzer": "standard",
-      "similarity": "BM25",
-      "fielddata": {
-          "format": "paged_bytes"
-      }
+      "similarity": "BM25"
     }
   }
 }
diff --git a/dev-tools/create_bwc_index.py b/dev-tools/create_bwc_index.py
index c8ade7f866a..361934908ec 100644
--- a/dev-tools/create_bwc_index.py
+++ b/dev-tools/create_bwc_index.py
@@ -247,6 +247,25 @@ def generate_index(client, version, index_name):
       }
     }
 
+  mappings['norms'] = {
+    'properties': {
+      'string_with_norms_disabled': {
+        'type': 'string',
+        'norms': {
+          'enabled': False
+        }
+      },
+      'string_with_norms_enabled': {
+        'type': 'string',
+        'index': 'not_analyzed',
+        'norms': {
+          'enabled': True,
+          'loading': 'eager'
+        }
+      }
+    }
+  }
+
   mappings['doc'] = {
     'properties': {
       'string': {
diff --git a/dev-tools/es_release_notes.pl b/dev-tools/es_release_notes.pl
index c3e93f91076..1a7565bfc56 100755
--- a/dev-tools/es_release_notes.pl
+++ b/dev-tools/es_release_notes.pl
@@ -29,7 +29,7 @@ my $Issue_URL  = "http://github.com/${User_Repo}issues/";
 
 my @Groups = qw(
     breaking deprecation feature
-    enhancement bug regression upgrade build doc test
+    enhancement bug regression upgrade non-issue build docs test
 );
 my %Group_Labels = (
     breaking    => 'Breaking changes',
@@ -42,6 +42,7 @@ my %Group_Labels = (
     regression  => 'Regressions',
     test        => 'Tests',
     upgrade     => 'Upgrades',
+    "non-issue" => 'Non-issue',
     other       => 'NOT CLASSIFIED',
 );
 
@@ -157,6 +158,8 @@ sub fetch_issues {
 ISSUE:
     for my $issue (@issues) {
         next if $seen{ $issue->{number} } && !$issue->{pull_request};
+        # uncomment for including/excluding PRs already issued in other versions
+        # next if grep {$_->{name}=~/^v2/} @{$issue->{labels}};
         my %labels = map { $_->{name} => 1 } @{ $issue->{labels} };
         my ($header) = map { substr( $_, 1 ) } grep {/^:/} keys %labels;
         $header ||= 'NOT CLASSIFIED';
diff --git a/distribution/build.gradle b/distribution/build.gradle
index d70f0254f3b..b9b2784a5b3 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -34,7 +34,7 @@ buildscript {
     }
   }
   dependencies {
-    classpath 'com.netflix.nebula:gradle-ospackage-plugin:3.1.0'
+    classpath 'com.netflix.nebula:gradle-ospackage-plugin:3.4.0'
   }
 }
 
@@ -337,21 +337,19 @@ configure(subprojects.findAll { ['deb', 'rpm'].contains(it.name) }) {
     /**
      * Suck up all the empty directories that we need to install into the path.
      */
-    Closure suckUpEmptyDirectories = { path ->
+    Closure suckUpEmptyDirectories = { path, u, g ->
       into(path) {
         from "${packagingFiles}/${path}"
         includeEmptyDirs true
         createDirectoryEntry true
-        /* All of these empty directories have this ownership. We're just
-          lucky! */
-        user 'elasticsearch'
-        permissionGroup 'elasticsearch'
+        user u
+        permissionGroup g
       }
     }
-    suckUpEmptyDirectories('/var/run')
-    suckUpEmptyDirectories('/var/log')
-    suckUpEmptyDirectories('/var/lib')
-    suckUpEmptyDirectories('/usr/share/elasticsearch')
+    suckUpEmptyDirectories('/var/run', 'elasticsearch', 'elasticsearch')
+    suckUpEmptyDirectories('/var/log', 'elasticsearch', 'elasticsearch')
+    suckUpEmptyDirectories('/var/lib', 'elasticsearch', 'elasticsearch')
+    suckUpEmptyDirectories('/usr/share/elasticsearch', 'root', 'root')
   }
 }
 
diff --git a/distribution/deb/src/main/packaging/init.d/elasticsearch b/distribution/deb/src/main/packaging/init.d/elasticsearch
index be45c45d192..1476a520c1d 100755
--- a/distribution/deb/src/main/packaging/init.d/elasticsearch
+++ b/distribution/deb/src/main/packaging/init.d/elasticsearch
@@ -99,7 +99,7 @@ fi
 # Define other required variables
 PID_FILE="$PID_DIR/$NAME.pid"
 DAEMON=$ES_HOME/bin/elasticsearch
-DAEMON_OPTS="-d -p $PID_FILE --default.path.home=$ES_HOME --default.path.logs=$LOG_DIR --default.path.data=$DATA_DIR --default.path.conf=$CONF_DIR"
+DAEMON_OPTS="-d -p $PID_FILE -Ees.default.path.logs=$LOG_DIR -Ees.default.path.data=$DATA_DIR -Ees.default.path.conf=$CONF_DIR"
 
 export ES_HEAP_SIZE
 export ES_HEAP_NEWSIZE
@@ -107,6 +107,7 @@ export ES_DIRECT_SIZE
 export ES_JAVA_OPTS
 export ES_GC_LOG_FILE
 export JAVA_HOME
+export ES_INCLUDE
 
 # Check DAEMON exists
 test -x $DAEMON || exit 0
diff --git a/distribution/licenses/commons-cli-1.3.1.jar.sha1 b/distribution/licenses/commons-cli-1.3.1.jar.sha1
deleted file mode 100644
index fc366d027f5..00000000000
--- a/distribution/licenses/commons-cli-1.3.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1303efbc4b181e5a58bf2e967dc156a3132b97c0
diff --git a/distribution/licenses/jopt-simple-4.9.jar.sha1 b/distribution/licenses/jopt-simple-4.9.jar.sha1
new file mode 100644
index 00000000000..b86fa62ac20
--- /dev/null
+++ b/distribution/licenses/jopt-simple-4.9.jar.sha1
@@ -0,0 +1 @@
+ee9e9eaa0a35360dcfeac129ff4923215fd65904
\ No newline at end of file
diff --git a/distribution/licenses/jopt-simple-LICENSE.txt b/distribution/licenses/jopt-simple-LICENSE.txt
new file mode 100644
index 00000000000..85f923a9526
--- /dev/null
+++ b/distribution/licenses/jopt-simple-LICENSE.txt
@@ -0,0 +1,24 @@
+/*
+ The MIT License
+
+ Copyright (c) 2004-2015 Paul R. Holser, Jr.
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
diff --git a/distribution/licenses/jopt-simple-NOTICE.txt b/distribution/licenses/jopt-simple-NOTICE.txt
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/distribution/licenses/lucene-analyzers-common-6.0.0-snapshot-bea235f.jar.sha1 b/distribution/licenses/lucene-analyzers-common-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index 74d21bae946..00000000000
--- a/distribution/licenses/lucene-analyzers-common-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3510af19947deadd929123aaf14d69b4bdec759a
\ No newline at end of file
diff --git a/distribution/licenses/lucene-analyzers-common-6.0.0-snapshot-f0aa4fc.jar.sha1 b/distribution/licenses/lucene-analyzers-common-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..2ed6eb6ef56
--- /dev/null
+++ b/distribution/licenses/lucene-analyzers-common-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+cd2388adc4b33c7530bbb8cd386e5c8c5c8e6aca
\ No newline at end of file
diff --git a/distribution/licenses/lucene-backward-codecs-6.0.0-snapshot-bea235f.jar.sha1 b/distribution/licenses/lucene-backward-codecs-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index ee6143bec14..00000000000
--- a/distribution/licenses/lucene-backward-codecs-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-247ad7c17cb7c742d7a9abd5d9980e4fab815178
\ No newline at end of file
diff --git a/distribution/licenses/lucene-backward-codecs-6.0.0-snapshot-f0aa4fc.jar.sha1 b/distribution/licenses/lucene-backward-codecs-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..28cdb1db9b1
--- /dev/null
+++ b/distribution/licenses/lucene-backward-codecs-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+f5bbdd01b98fab7c18b46e762de3e39221b0c8fc
\ No newline at end of file
diff --git a/distribution/licenses/lucene-core-6.0.0-snapshot-bea235f.jar.sha1 b/distribution/licenses/lucene-core-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index 2d39f84d21e..00000000000
--- a/distribution/licenses/lucene-core-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c0712dbec58abad545646edab67d58f7373f5329
\ No newline at end of file
diff --git a/distribution/licenses/lucene-core-6.0.0-snapshot-f0aa4fc.jar.sha1 b/distribution/licenses/lucene-core-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..c304106975b
--- /dev/null
+++ b/distribution/licenses/lucene-core-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+18ad74518b34af7cfbd6c1e3a408920ff7665501
\ No newline at end of file
diff --git a/distribution/licenses/lucene-grouping-6.0.0-snapshot-bea235f.jar.sha1 b/distribution/licenses/lucene-grouping-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index a3ce82c8a04..00000000000
--- a/distribution/licenses/lucene-grouping-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7573e3efb12dd16fdc991edaf408877dab20c030
\ No newline at end of file
diff --git a/distribution/licenses/lucene-grouping-6.0.0-snapshot-f0aa4fc.jar.sha1 b/distribution/licenses/lucene-grouping-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..a95cc29cc7d
--- /dev/null
+++ b/distribution/licenses/lucene-grouping-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+dc0b211e31b8f1e0ee3a9e8f9c71b13fa088dabf
\ No newline at end of file
diff --git a/distribution/licenses/lucene-highlighter-6.0.0-snapshot-bea235f.jar.sha1 b/distribution/licenses/lucene-highlighter-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index 9259a2c66c1..00000000000
--- a/distribution/licenses/lucene-highlighter-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-96ef0a9a43a5fc99d27bb7e7d61517ee4c7e54a4
\ No newline at end of file
diff --git a/distribution/licenses/lucene-highlighter-6.0.0-snapshot-f0aa4fc.jar.sha1 b/distribution/licenses/lucene-highlighter-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..8f57bb02639
--- /dev/null
+++ b/distribution/licenses/lucene-highlighter-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+bbd503396c08546f1b9e023e77dbf25bbb052d1c
\ No newline at end of file
diff --git a/distribution/licenses/lucene-join-6.0.0-snapshot-bea235f.jar.sha1 b/distribution/licenses/lucene-join-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index 4959f5f163c..00000000000
--- a/distribution/licenses/lucene-join-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d93de34947d37e31a337cdfed400333588c378d8
\ No newline at end of file
diff --git a/distribution/licenses/lucene-join-6.0.0-snapshot-f0aa4fc.jar.sha1 b/distribution/licenses/lucene-join-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..835bac49233
--- /dev/null
+++ b/distribution/licenses/lucene-join-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+96fd93d4a4192c42b0d56198b73a25440d4db2f7
\ No newline at end of file
diff --git a/distribution/licenses/lucene-memory-6.0.0-snapshot-bea235f.jar.sha1 b/distribution/licenses/lucene-memory-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index 5218d0a019e..00000000000
--- a/distribution/licenses/lucene-memory-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9c292930b1828e68f06509944a5346c141d56fd4
\ No newline at end of file
diff --git a/distribution/licenses/lucene-memory-6.0.0-snapshot-f0aa4fc.jar.sha1 b/distribution/licenses/lucene-memory-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..1e392d3e246
--- /dev/null
+++ b/distribution/licenses/lucene-memory-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+ddd44a319d201ff73cd25be139bd3175226ab5a5
\ No newline at end of file
diff --git a/distribution/licenses/lucene-misc-6.0.0-snapshot-bea235f.jar.sha1 b/distribution/licenses/lucene-misc-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index 947722edfd3..00000000000
--- a/distribution/licenses/lucene-misc-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-866ed93f48683e877ffa4d9baa1323dcffbc65d7
\ No newline at end of file
diff --git a/distribution/licenses/lucene-misc-6.0.0-snapshot-f0aa4fc.jar.sha1 b/distribution/licenses/lucene-misc-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..a21aaef33f5
--- /dev/null
+++ b/distribution/licenses/lucene-misc-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+07d943ecdc552632bdca8f2772fd081a02cbf589
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queries-6.0.0-snapshot-bea235f.jar.sha1 b/distribution/licenses/lucene-queries-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index 6caf86a6b96..00000000000
--- a/distribution/licenses/lucene-queries-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-967d9c2647bdd4d88961747f7436a5a92aa0385b
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queries-6.0.0-snapshot-f0aa4fc.jar.sha1 b/distribution/licenses/lucene-queries-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..57fb022de53
--- /dev/null
+++ b/distribution/licenses/lucene-queries-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+66c72fd979f54480af75d01719ef25da62c0a8b6
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queryparser-6.0.0-snapshot-bea235f.jar.sha1 b/distribution/licenses/lucene-queryparser-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index b3e92d3f168..00000000000
--- a/distribution/licenses/lucene-queryparser-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-981030d83a7504267f3141d7365fad9b46d51465
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queryparser-6.0.0-snapshot-f0aa4fc.jar.sha1 b/distribution/licenses/lucene-queryparser-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..5c311c4bd9b
--- /dev/null
+++ b/distribution/licenses/lucene-queryparser-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+8992204f922fe52af557e691cbfb4c54f92b76bd
\ No newline at end of file
diff --git a/distribution/licenses/lucene-sandbox-6.0.0-snapshot-bea235f.jar.sha1 b/distribution/licenses/lucene-sandbox-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index 7b5176c4963..00000000000
--- a/distribution/licenses/lucene-sandbox-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-707691b1baf22c29020569f5b875d200a4955411
\ No newline at end of file
diff --git a/distribution/licenses/lucene-sandbox-6.0.0-snapshot-f0aa4fc.jar.sha1 b/distribution/licenses/lucene-sandbox-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..20f0037ea31
--- /dev/null
+++ b/distribution/licenses/lucene-sandbox-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+8565264e00bc43e1226ff0d2e986dbb26d353ce2
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-6.0.0-snapshot-bea235f.jar.sha1 b/distribution/licenses/lucene-spatial-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index 9df2a16b886..00000000000
--- a/distribution/licenses/lucene-spatial-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-be9e78130a069983f611f484d5b7b87bda0d6370
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-6.0.0-snapshot-f0aa4fc.jar.sha1 b/distribution/licenses/lucene-spatial-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..6a909857945
--- /dev/null
+++ b/distribution/licenses/lucene-spatial-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+98fc1bb7e005f33c388be66486341ad8168b72eb
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-extras-6.0.0-snapshot-bea235f.jar.sha1 b/distribution/licenses/lucene-spatial-extras-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index 6badc36d361..00000000000
--- a/distribution/licenses/lucene-spatial-extras-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-edeef6ce8a58d5e6a074bebf545918d04e8579e1
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-extras-6.0.0-snapshot-f0aa4fc.jar.sha1 b/distribution/licenses/lucene-spatial-extras-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..b741ccd62a7
--- /dev/null
+++ b/distribution/licenses/lucene-spatial-extras-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+b5b651b0adbc2f404e091817282dabd7b432c677
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial3d-6.0.0-snapshot-bea235f.jar.sha1 b/distribution/licenses/lucene-spatial3d-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index 480ae590aed..00000000000
--- a/distribution/licenses/lucene-spatial3d-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d86a7ba859576bdcee1dacd8f407ccf71f982c60
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial3d-6.0.0-snapshot-f0aa4fc.jar.sha1 b/distribution/licenses/lucene-spatial3d-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..512e4b7b592
--- /dev/null
+++ b/distribution/licenses/lucene-spatial3d-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+334e194bf83c75f0ae165e3e72b6fa35c5d636c5
\ No newline at end of file
diff --git a/distribution/licenses/lucene-suggest-6.0.0-snapshot-bea235f.jar.sha1 b/distribution/licenses/lucene-suggest-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index 7835298c4a2..00000000000
--- a/distribution/licenses/lucene-suggest-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a3860de6502576f142dc948eb2005fa4dc0c27c5
\ No newline at end of file
diff --git a/distribution/licenses/lucene-suggest-6.0.0-snapshot-f0aa4fc.jar.sha1 b/distribution/licenses/lucene-suggest-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..3d2cf156d40
--- /dev/null
+++ b/distribution/licenses/lucene-suggest-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+89c46e9601cf8fb9acf77398838f8710c9e44053
\ No newline at end of file
diff --git a/distribution/rpm/build.gradle b/distribution/rpm/build.gradle
index 2ab78fe7e41..177ab518a10 100644
--- a/distribution/rpm/build.gradle
+++ b/distribution/rpm/build.gradle
@@ -25,7 +25,14 @@ task buildRpm(type: Rpm) {
   packageGroup 'Application/Internet'
   prefix '/usr'
   packager 'Elasticsearch'
-  release '1'
+  if (version.contains('~')) {
+    def tokenized = version.tokenize('~')
+    version tokenized[0]
+    release tokenized[1]
+  } else {
+    version version
+    release '1'
+  }
   arch NOARCH
   os LINUX
   // TODO ospackage doesn't support icon but we used to have one
diff --git a/distribution/rpm/src/main/packaging/init.d/elasticsearch b/distribution/rpm/src/main/packaging/init.d/elasticsearch
index 12fed7dbc33..c68a5b65f3f 100644
--- a/distribution/rpm/src/main/packaging/init.d/elasticsearch
+++ b/distribution/rpm/src/main/packaging/init.d/elasticsearch
@@ -66,6 +66,7 @@ export ES_JAVA_OPTS
 export ES_GC_LOG_FILE
 export ES_STARTUP_SLEEP_TIME
 export JAVA_HOME
+export ES_INCLUDE
 
 lockfile=/var/lock/subsys/$prog
 
@@ -116,7 +117,7 @@ start() {
     cd $ES_HOME
     echo -n $"Starting $prog: "
     # if not running, start it up here, usually something like "daemon $exec"
-    daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Des.default.path.home=$ES_HOME -Des.default.path.logs=$LOG_DIR -Des.default.path.data=$DATA_DIR -Des.default.path.conf=$CONF_DIR
+    daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Ees.default.path.home=$ES_HOME -Ees.default.path.logs=$LOG_DIR -Ees.default.path.data=$DATA_DIR -Ees.default.path.conf=$CONF_DIR
     retval=$?
     echo
     [ $retval -eq 0 ] && touch $lockfile
diff --git a/distribution/src/main/packaging/scripts/postinst b/distribution/src/main/packaging/scripts/postinst
index 61ac5f27cd7..4dd5bbf5288 100644
--- a/distribution/src/main/packaging/scripts/postinst
+++ b/distribution/src/main/packaging/scripts/postinst
@@ -96,4 +96,8 @@ elif [ "$RESTART_ON_UPGRADE" = "true" ]; then
     echo " OK"
 fi
 
+chown -R $ES_USER:$ES_GROUP /var/lib/elasticsearch
+chown -R $ES_USER:$ES_GROUP /var/log/elasticsearch
+chown -R root:$ES_GROUP /etc/elasticsearch
+
 ${scripts.footer}
diff --git a/distribution/src/main/packaging/systemd/elasticsearch.service b/distribution/src/main/packaging/systemd/elasticsearch.service
index 301586c1038..1aed30ac968 100644
--- a/distribution/src/main/packaging/systemd/elasticsearch.service
+++ b/distribution/src/main/packaging/systemd/elasticsearch.service
@@ -20,11 +20,10 @@ Group=elasticsearch
 ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec
 
 ExecStart=/usr/share/elasticsearch/bin/elasticsearch \
-                                                -Des.pidfile=${PID_DIR}/elasticsearch.pid \
-                                                -Des.default.path.home=${ES_HOME} \
-                                                -Des.default.path.logs=${LOG_DIR} \
-                                                -Des.default.path.data=${DATA_DIR} \
-                                                -Des.default.path.conf=${CONF_DIR}
+                                                -p ${PID_DIR}/elasticsearch.pid \
+                                                -Ees.default.path.logs=${LOG_DIR} \
+                                                -Ees.default.path.data=${DATA_DIR} \
+                                                -Ees.default.path.conf=${CONF_DIR}
 
 StandardOutput=journal
 StandardError=inherit
diff --git a/distribution/src/main/resources/bin/elasticsearch b/distribution/src/main/resources/bin/elasticsearch
index b15105f1854..253ee1ee1f5 100755
--- a/distribution/src/main/resources/bin/elasticsearch
+++ b/distribution/src/main/resources/bin/elasticsearch
@@ -127,10 +127,10 @@ export HOSTNAME
 daemonized=`echo $* | egrep -- '(^-d |-d$| -d |--daemonize$|--daemonize )'`
 if [ -z "$daemonized" ] ; then
     exec "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS -Des.path.home="$ES_HOME" -cp "$ES_CLASSPATH" \
-          org.elasticsearch.bootstrap.Elasticsearch start "$@"
+          org.elasticsearch.bootstrap.Elasticsearch "$@"
 else
     exec "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS -Des.path.home="$ES_HOME" -cp "$ES_CLASSPATH" \
-          org.elasticsearch.bootstrap.Elasticsearch start "$@" <&- &
+          org.elasticsearch.bootstrap.Elasticsearch "$@" <&- &
     retval=$?
     pid=$!
     [ $retval -eq 0 ] || exit $retval
diff --git a/distribution/src/main/resources/bin/elasticsearch.bat b/distribution/src/main/resources/bin/elasticsearch.bat
index a0079fc967c..a2e5d45e1c4 100644
--- a/distribution/src/main/resources/bin/elasticsearch.bat
+++ b/distribution/src/main/resources/bin/elasticsearch.bat
@@ -43,6 +43,6 @@ IF ERRORLEVEL 1 (
 	EXIT /B %ERRORLEVEL%
 )
 
-"%JAVA_HOME%\bin\java" %JAVA_OPTS% %ES_JAVA_OPTS% %ES_PARAMS% !newparams! -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch" start
+"%JAVA_HOME%\bin\java" %JAVA_OPTS% %ES_JAVA_OPTS% %ES_PARAMS% -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch" !newparams!
 
 ENDLOCAL
diff --git a/distribution/src/main/resources/bin/elasticsearch.in.bat b/distribution/src/main/resources/bin/elasticsearch.in.bat
index 7138cf5f5ca..537df9d4f9f 100644
--- a/distribution/src/main/resources/bin/elasticsearch.in.bat
+++ b/distribution/src/main/resources/bin/elasticsearch.in.bat
@@ -85,6 +85,10 @@ REM JAVA_OPTS=%JAVA_OPTS% -XX:HeapDumpPath=$ES_HOME/logs/heapdump.hprof
 REM Disables explicit GC
 set JAVA_OPTS=%JAVA_OPTS% -XX:+DisableExplicitGC
 
+REM Enable pre-touching of memory pages used by the JVM during hotspot
+REM initialization
+set JAVA_OPTS=%JAVA_OPTS% -XX:+AlwaysPreTouch
+
 REM Ensure UTF-8 encoding by default (e.g. filenames)
 set JAVA_OPTS=%JAVA_OPTS% -Dfile.encoding=UTF-8
 
@@ -100,4 +104,4 @@ ECHO additional elements via the plugin mechanism, or if code must really be 1>&
 ECHO added to the main classpath, add jars to lib\, unsupported 1>&2
 EXIT /B 1
 )
-set ES_PARAMS=-Delasticsearch -Des-foreground=yes -Des.path.home="%ES_HOME%"
+set ES_PARAMS=-Delasticsearch -Des.path.home="%ES_HOME%"
diff --git a/distribution/src/main/resources/bin/elasticsearch.in.sh b/distribution/src/main/resources/bin/elasticsearch.in.sh
index f859a06ffab..69d2fc94112 100644
--- a/distribution/src/main/resources/bin/elasticsearch.in.sh
+++ b/distribution/src/main/resources/bin/elasticsearch.in.sh
@@ -81,6 +81,10 @@ JAVA_OPTS="$JAVA_OPTS -XX:+HeapDumpOnOutOfMemoryError"
 # Disables explicit GC
 JAVA_OPTS="$JAVA_OPTS -XX:+DisableExplicitGC"
 
+# Enable pre-touching of memory pages used by the JVM during hotspot
+# initialization
+JAVA_OPTS="$JAVA_OPTS -XX:+AlwaysPreTouch"
+
 # Ensure UTF-8 encoding by default (e.g. filenames)
 JAVA_OPTS="$JAVA_OPTS -Dfile.encoding=UTF-8"
 
diff --git a/distribution/src/main/resources/bin/service.bat b/distribution/src/main/resources/bin/service.bat
index 22242e36ff9..2786c87a634 100644
--- a/distribution/src/main/resources/bin/service.bat
+++ b/distribution/src/main/resources/bin/service.bat
@@ -152,7 +152,7 @@ if "%DATA_DIR%" == "" set DATA_DIR=%ES_HOME%\data
 
 if "%CONF_DIR%" == "" set CONF_DIR=%ES_HOME%\config
 
-set ES_PARAMS=-Delasticsearch;-Des.path.home="%ES_HOME%";-Des.default.path.home="%ES_HOME%";-Des.default.path.logs="%LOG_DIR%";-Des.default.path.data="%DATA_DIR%";-Des.default.path.conf="%CONF_DIR%"
+set ES_PARAMS=-Delasticsearch;-Des.path.home="%ES_HOME%";-Des.default.path.logs="%LOG_DIR%";-Des.default.path.data="%DATA_DIR%";-Des.default.path.conf="%CONF_DIR%"
 
 set JVM_OPTS=%JAVA_OPTS: =;%
 
diff --git a/distribution/src/main/resources/config/logging.yml b/distribution/src/main/resources/config/logging.yml
index 939aa1eed0e..187e79cffa0 100644
--- a/distribution/src/main/resources/config/logging.yml
+++ b/distribution/src/main/resources/config/logging.yml
@@ -1,4 +1,4 @@
-# you can override this using by setting a system property, for example -Des.logger.level=DEBUG
+# you can override this using by setting a system property, for example -Ees.logger.level=DEBUG
 es.logger.level: INFO
 rootLogger: ${es.logger.level}, console, file
 logger:
diff --git a/docs/java-api/client.asciidoc b/docs/java-api/client.asciidoc
index 7eea693d432..7af8c230269 100644
--- a/docs/java-api/client.asciidoc
+++ b/docs/java-api/client.asciidoc
@@ -12,12 +12,16 @@ Obtaining an elasticsearch `Client` is simple. The most common way to
 get a client is by creating a <<transport-client,`TransportClient`>>
 that connects to a cluster.
 
-*Important:*
-______________________________________________________________________________________________________________________________________________________________
-Please note that you are encouraged to use the same version on client
-and cluster sides. You may hit some incompatibility issues when mixing
-major versions.
-______________________________________________________________________________________________________________________________________________________________
+[IMPORTANT]
+==============================
+
+The client must have the same major version (e.g. `2.x`, or `5.x`) as the
+nodes in the cluster. Clients may connect to clusters which have a different
+minor version (e.g. `2.3.x`) but it is possible that new funcionality may not
+be supported.  Ideally, the client should have the same version as the
+cluster.
+
+==============================
 
 
 [[transport-client]]
@@ -53,11 +57,23 @@ Client client = TransportClient.builder().settings(settings).build();
 //Add transport addresses and do something with the client...
 --------------------------------------------------
 
-The client allows sniffing the rest of the cluster, which adds data nodes
-into its list of machines to use. In this case, note that the IP addresses
-used will be the ones that the other nodes were started with (the
-"publish" address). In order to enable it, set the
-`client.transport.sniff` to `true`:
+The Transport client comes with a cluster sniffing feature which
+allows it to dynamically add new hosts and remove old ones.
+When sniffing is enabled the the transport client will connect to the nodes in its
+internal node list, which is built via calls to addTransportAddress.
+After this, the client will call the internal cluster state API on those nodes
+to discover available data nodes. The internal node list of the client will
+be replaced with those data nodes only. This list is refreshed every five seconds by default.
+Note that the IP addresses the sniffer connects to are the ones declared as the 'publish'
+address in those node's elasticsearch config.
+
+Keep in mind that list might possibly not include the original node it connected to
+if that node is not a data node. If, for instance, you initially connect to a
+master node, after sniffing no further requests will go to that master node,
+but rather to any data nodes instead. The reason the transport excludes non-data
+nodes is to avoid sending search traffic to master only nodes.
+
+In order to enable sniffing, set `client.transport.sniff` to `true`:
 
 [source,java]
 --------------------------------------------------
diff --git a/docs/java-api/index.asciidoc b/docs/java-api/index.asciidoc
index 012633f1e4b..fbd3fe33165 100644
--- a/docs/java-api/index.asciidoc
+++ b/docs/java-api/index.asciidoc
@@ -142,8 +142,6 @@ include::search.asciidoc[]
 
 include::aggs.asciidoc[]
 
-include::percolate.asciidoc[]
-
 include::query-dsl.asciidoc[]
 
 include::indexed-scripts.asciidoc[]
diff --git a/docs/java-api/percolate.asciidoc b/docs/java-api/query-dsl/percolator-query.asciidoc
similarity index 63%
rename from docs/java-api/percolate.asciidoc
rename to docs/java-api/query-dsl/percolator-query.asciidoc
index a08e09b2afe..04f3ba9cb8d 100644
--- a/docs/java-api/percolate.asciidoc
+++ b/docs/java-api/query-dsl/percolator-query.asciidoc
@@ -1,12 +1,8 @@
-[[percolate]]
-== Percolate API
+[[java-query-percolator-query]]
+==== Percolator query
 
-The percolator allows one to register queries against an index, and then
-send `percolate` requests which include a doc, getting back the
-queries that match on that doc out of the set of registered queries.
-
-Read the main {ref}/search-percolate.html[percolate]
-documentation before reading this guide.
+See:
+ * {ref}/query-dsl-percolator-query.html[Percolator Query]
 
 [source,java]
 --------------------------------------------------
@@ -37,14 +33,12 @@ docBuilder.field("doc").startObject(); //This is needed to designate the documen
 docBuilder.field("content", "This is amazing!");
 docBuilder.endObject(); //End of the doc field
 docBuilder.endObject(); //End of the JSON root object
-//Percolate
-PercolateResponse response = client.preparePercolate()
-                        .setIndices("myIndexName")
-                        .setDocumentType("myDocumentType")
-                        .setSource(docBuilder).execute().actionGet();
+// Percolate, by executing the percolator query in the query dsl:
+SearchResponse response = client().prepareSearch("myIndexName")
+        .setQuery(QueryBuilders.percolatorQuery("myDocumentType", docBuilder.bytes()))
+        .get();
 //Iterate over the results
-for(PercolateResponse.Match match : response) {
-    //Handle the result which is the name of
-    //the query in the percolator
+for(SearchHit hit : response.getHits()) {
+    // Percolator queries as hit
 }
 --------------------------------------------------
diff --git a/docs/java-api/query-dsl/special-queries.asciidoc b/docs/java-api/query-dsl/special-queries.asciidoc
index 0df5af890ec..d5c9db07391 100644
--- a/docs/java-api/query-dsl/special-queries.asciidoc
+++ b/docs/java-api/query-dsl/special-queries.asciidoc
@@ -27,3 +27,5 @@ include::template-query.asciidoc[]
 
 include::script-query.asciidoc[]
 
+include::percolator-query.asciidoc[]
+
diff --git a/docs/plugins/analysis-kuromoji.asciidoc b/docs/plugins/analysis-kuromoji.asciidoc
index 8d1d6ac2d98..ff42287d03f 100644
--- a/docs/plugins/analysis-kuromoji.asciidoc
+++ b/docs/plugins/analysis-kuromoji.asciidoc
@@ -122,6 +122,28 @@ dictionary to `$ES_HOME/config/userdict_ja.txt`:
 東京スカイツリー,東京 スカイツリー,トウキョウ スカイツリー,カスタム名詞
 -----------------------
 
+`nbest_cost`/`nbest_examples`::
++
+--
+Additional expert user parameters `nbest_cost` and `nbest_examples` can be used
+to include additional tokens that most likely according to the statistical model.
+If both parameters are used, the largest number of both is applied.
+
+`nbest_cost`::
+
+    The `nbest_cost` parameter specifies an additional Viterbi cost.
+    The KuromojiTokenizer will include all tokens in Viterbi paths that are
+    within the nbest_cost value of the best path.
+
+`nbest_examples`::
+
+    The `nbest_examples` can be used to find a `nbest_cost` value based on examples.
+    For example, a value of /箱根山-箱根/成田空港-成田/ indicates that in the texts,
+    箱根山 (Mt. Hakone) and 成田空港 (Narita Airport) we'd like a cost that gives is us
+    箱根 (Hakone) and 成田 (Narita).
+--
+
+
 Then create an analyzer as follows:
 
 [source,json]
@@ -452,3 +474,48 @@ The above request returns:
 }
 --------------------------------------------------
 
+[[analysis-kuromoji-number]]
+===== `kuromoji_number` token filter
+
+The `kuromoji_number` token filter normalizes Japanese numbers (kansūji)
+to regular Arabic decimal numbers in half-width characters.
+
+[source,json]
+--------------------------------------------------
+PUT kuromoji_sample
+{
+  "settings": {
+    "index": {
+      "analysis": {
+        "analyzer": {
+          "my_analyzer": {
+            "tokenizer": "kuromoji_tokenizer",
+            "filter": [
+              "kuromoji_number"
+            ]
+          }
+        }
+      }
+    }
+  }
+}
+
+POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=一〇〇〇
+
+--------------------------------------------------
+// AUTOSENSE
+
+[source,text]
+--------------------------------------------------
+# Result
+{
+  "tokens" : [ {
+    "token" : "1000",
+    "start_offset" : 0,
+    "end_offset" : 4,
+    "type" : "word",
+    "position" : 1
+  } ]
+}
+--------------------------------------------------
+
diff --git a/docs/plugins/authors.asciidoc b/docs/plugins/authors.asciidoc
index d0a606987ae..af3710f7e2d 100644
--- a/docs/plugins/authors.asciidoc
+++ b/docs/plugins/authors.asciidoc
@@ -26,9 +26,7 @@ https://github.com/elastic/elasticsearch/blob/master/buildSrc/src/main/resources
 Either fill in this template yourself (see
 https://github.com/lmenezes/elasticsearch-kopf/blob/master/plugin-descriptor.properties[elasticsearch-kopf]
 as an example) or, if you are using Elasticsearch's Gradle build system, you
-can fill in the necessary values in the `build.gradle` file for your plugin. For
-instance, see
-https://github.com/elastic/elasticsearch/blob/master/plugins/site-example/build.gradle[`/plugins/site-example/build.gradle`].
+can fill in the necessary values in the `build.gradle` file for your plugin.
 
 [float]
 ==== Mandatory elements for plugins
diff --git a/docs/plugins/index.asciidoc b/docs/plugins/index.asciidoc
index 598e7872dcd..fa05668c131 100644
--- a/docs/plugins/index.asciidoc
+++ b/docs/plugins/index.asciidoc
@@ -67,3 +67,6 @@ include::integrations.asciidoc[]
 
 include::authors.asciidoc[]
 
+include::redirects.asciidoc[]
+
+
diff --git a/docs/plugins/mapper-attachments.asciidoc b/docs/plugins/mapper-attachments.asciidoc
index ed992623a50..e65ab9ff64a 100644
--- a/docs/plugins/mapper-attachments.asciidoc
+++ b/docs/plugins/mapper-attachments.asciidoc
@@ -176,7 +176,7 @@ need to specify the `type` (like `string` or `date`) since it is already known.
 [[mapper-attachments-copy-to]]
 ==== Copy To feature
 
-If you want to use http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html#copy-to[copy_to]
+If you want to use https://www.elastic.co/guide/en/elasticsearch/reference/current/copy-to.html[copy_to]
 feature, you need to define it on each sub-field you want to copy to another field:
 
 [source,js]
@@ -405,41 +405,3 @@ It gives back:
    }
 }
 --------------------------
-
-[[mapper-attachments-standalone]]
-==== Stand alone runner
-
-If you want to run some tests within your IDE, you can use `StandaloneRunner` class.
-It accepts arguments:
-
-* `-u file://URL/TO/YOUR/DOC`
-* `--size` set extracted size (default to mapper attachment size)
-* `BASE64` encoded binary
-
-Example:
-
-[source,sh]
---------------------------
-StandaloneRunner BASE64Text
-StandaloneRunner -u /tmp/mydoc.pdf
-StandaloneRunner -u /tmp/mydoc.pdf --size 1000000
---------------------------
-
-It produces something like:
-
-[source,text]
---------------------------
-## Extracted text
---------------------- BEGIN -----------------------
-This is the extracted text
----------------------- END ------------------------
-## Metadata
-- author: null
-- content_length: null
-- content_type: application/pdf
-- date: null
-- keywords: null
-- language: null
-- name: null
-- title: null
---------------------------
diff --git a/docs/plugins/plugin-script.asciidoc b/docs/plugins/plugin-script.asciidoc
index d2e57b2efc8..fba4704ab97 100644
--- a/docs/plugins/plugin-script.asciidoc
+++ b/docs/plugins/plugin-script.asciidoc
@@ -167,7 +167,7 @@ can do this as follows:
 
 [source,sh]
 ---------------------
-sudo bin/elasticsearch-plugin -Des.path.conf=/path/to/custom/config/dir install <plugin name>
+sudo bin/elasticsearch-plugin -Ees.path.conf=/path/to/custom/config/dir install <plugin name>
 ---------------------
 
 You can also set the `CONF_DIR` environment variable to the custom config
diff --git a/docs/plugins/redirects.asciidoc b/docs/plugins/redirects.asciidoc
new file mode 100644
index 00000000000..caf2008e521
--- /dev/null
+++ b/docs/plugins/redirects.asciidoc
@@ -0,0 +1,40 @@
+["appendix",role="exclude",id="redirects"]
+= Deleted pages
+
+The following pages have moved or been deleted.
+
+[role="exclude",id="discovery-multicast"]
+=== Multicast Discovery Plugin
+
+The `multicast-discovery` plugin has been removed.  Instead, configure networking
+using unicast (see {ref}/modules-network.html[Network settings]) or using
+one of the <<discovery,cloud discovery plugins>>.
+
+[role="exclude",id="cloud-aws"]
+=== AWS Cloud Plugin
+
+The `cloud-aws` plugin has been split into two separate plugins:
+
+*  <<discovery-ec2>> (`discovery-ec2`)
+*  <<repository-s3>> (`repository-s3`)
+
+
+[role="exclude",id="cloud-azure"]
+=== Azure Cloud Plugin
+
+The `cloud-azure` plugin has been split into two separate plugins:
+
+*  <<discovery-azure>> (`discovery-azure`)
+*  <<repository-azure>> (`repository-azure`)
+
+
+[role="exclude",id="cloud-gce"]
+=== GCE Cloud Plugin
+
+The `cloud-gce` plugin has been renamed to <<discovery-gce>> (`discovery-gce`).
+
+
+
+
+
+
diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc
index 007018a6abc..8780b3a710e 100644
--- a/docs/plugins/repository-s3.asciidoc
+++ b/docs/plugins/repository-s3.asciidoc
@@ -189,7 +189,7 @@ The following settings are supported:
 
     Big files can be broken down into chunks during snapshotting if needed.
     The chunk size can be specified in bytes or by using size value notation,
-    i.e. `1g`, `10m`, `5k`. Defaults to `100m`.
+    i.e. `1gb`, `10mb`, `5kb`. Defaults to `1gb`.
 
 `compress`::
 
@@ -210,7 +210,7 @@ The following settings are supported:
     to split the chunk into several parts, each of `buffer_size` length, and
     to upload each part in its own request. Note that setting a buffer
     size lower than `5mb` is not allowed since it will prevents the use of the
-    Multipart API and may result in upload errors. Defaults to `5mb`.
+    Multipart API and may result in upload errors. Defaults to `100mb`.
 
 `max_retries`::
 
diff --git a/docs/reference/aggregations/bucket/daterange-aggregation.asciidoc b/docs/reference/aggregations/bucket/daterange-aggregation.asciidoc
index e649928810b..a69b2b3cb11 100644
--- a/docs/reference/aggregations/bucket/daterange-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/daterange-aggregation.asciidoc
@@ -118,7 +118,7 @@ Any characters in the pattern that are not in the ranges of ['a'..'z'] and ['A'.
 Dates can be converted from another time zone to UTC by specifying the `time_zone` parameter.
 
 Time zones may either be specified as an ISO 8601 UTC offset (e.g. +01:00 or -08:00) or as one of
-the the http://joda-time.sourceforge.net/timezones.html[time zone ids] from the TZ database.
+the http://www.joda.org/joda-time/timezones.html[time zone ids] from the TZ database.
 
 The `time_zone` parameter is also applied to rounding in date math expressions. As an example,
 to round to the beginning of the day in the CET time zone, you can do the following:
diff --git a/docs/reference/aggregations/bucket/nested-aggregation.asciidoc b/docs/reference/aggregations/bucket/nested-aggregation.asciidoc
index f5872bdc5dc..89142df13e7 100644
--- a/docs/reference/aggregations/bucket/nested-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/nested-aggregation.asciidoc
@@ -16,7 +16,7 @@ price for the product. The mapping could look like:
             "resellers" : { <1>
                 "type" : "nested",
                 "properties" : {
-                    "name" : { "type" : "string" },
+                    "name" : { "type" : "text" },
                     "price" : { "type" : "double" }
                 }
             }
diff --git a/docs/reference/aggregations/bucket/reverse-nested-aggregation.asciidoc b/docs/reference/aggregations/bucket/reverse-nested-aggregation.asciidoc
index 03bcdb0a18c..9dba1f2adf0 100644
--- a/docs/reference/aggregations/bucket/reverse-nested-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/reverse-nested-aggregation.asciidoc
@@ -22,12 +22,12 @@ the issue documents as nested documents. The mapping could look like:
 
     "issue" : {
         "properties" : {
-            "tags" : { "type" : "string" }
+            "tags" : { "type" : "text" }
             "comments" : { <1>
                 "type" : "nested"
                 "properties" : {
-                    "username" : { "type" : "string", "index" : "not_analyzed" },
-                    "comment" : { "type" : "string" }
+                    "username" : { "type" : "keyword" },
+                    "comment" : { "type" : "text" }
                 }
             }
         }
diff --git a/docs/reference/analysis/analyzers/keyword-analyzer.asciidoc b/docs/reference/analysis/analyzers/keyword-analyzer.asciidoc
index 7704895c9da..815037596cf 100644
--- a/docs/reference/analysis/analyzers/keyword-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/keyword-analyzer.asciidoc
@@ -4,4 +4,4 @@
 An analyzer of type `keyword` that "tokenizes" an entire stream as a
 single token. This is useful for data like zip codes, ids and so on.
 Note, when using mapping definitions, it might make more sense to simply
-mark the field as `not_analyzed`.
+map the field as a <<keyword,`keyword`>>.
diff --git a/docs/reference/analysis/tokenfilters/compound-word-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/compound-word-tokenfilter.asciidoc
index f77c47d156e..1268727b2ef 100644
--- a/docs/reference/analysis/tokenfilters/compound-word-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/compound-word-tokenfilter.asciidoc
@@ -27,7 +27,8 @@ use. For languages like German they are quite good.
 
 XML based hyphenation grammar files can be found in the
 http://offo.sourceforge.net/hyphenation/#FOP+XML+Hyphenation+Patterns[Objects For Formatting Objects]
-(OFFO) Sourceforge project. You can download http://downloads.sourceforge.net/offo/offo-hyphenation.zip[offo-hyphenation.zip]
+(OFFO) Sourceforge project. Currently only FOP v1.2 compatible hyphenation files
+are supported. You can download https://sourceforge.net/projects/offo/files/offo-hyphenation/1.2/offo-hyphenation_v1.2.zip/download[offo-hyphenation_v1.2.zip]
 directly and look in the `offo-hyphenation/hyph/` directory.
 Credits for the hyphenation code go to the Apache FOP project .
 
diff --git a/docs/reference/analysis/tokenizers/standard-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/standard-tokenizer.asciidoc
index c8b405bf820..42dbe5a864a 100644
--- a/docs/reference/analysis/tokenizers/standard-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/standard-tokenizer.asciidoc
@@ -13,6 +13,6 @@ type:
 |=======================================================================
 |Setting |Description
 |`max_token_length` |The maximum token length. If a token is seen that
-exceeds this length then it is discarded. Defaults to `255`.
+exceeds this length then it is split at `max_token_length` intervals. Defaults to `255`.
 |=======================================================================
 
diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc
index 90ff91513aa..6d4750da5a6 100644
--- a/docs/reference/cat/nodes.asciidoc
+++ b/docs/reference/cat/nodes.asciidoc
@@ -114,10 +114,18 @@ node (c) |d
 cache memory |0b
 |`fielddata.evictions` |`fe`, `fielddataEvictions` |No |Fielddata cache
 evictions |0
-|`filter_cache.memory_size` |`fcm`, `filterCacheMemory` |No |Used filter
+|`query_cache.memory_size` |`qcm`, `queryCacheMemory` |No |Used query
 cache memory |0b
-|`filter_cache.evictions` |`fce`, `filterCacheEvictions` |No |Filter
+|`query_cache.evictions` |`qce`, `queryCacheEvictions` |No |Query
 cache evictions |0
+|`request_cache.memory_size` |`rcm`, `requestCacheMemory` |No | Used request
+cache memory |0b
+|`request_cache.evictions` |`rce`, `requestCacheEvictions` |No |Request
+cache evictions |0
+|`request_cache.hit_count` |`rchc`, `requestCacheHitCount` |No | Request
+cache hit count |0
+|`request_cache.miss_count` |`rcmc`, `requestCacheMissCount` |No | Request
+cache miss count |0
 |`flush.total` |`ft`, `flushTotal` |No |Number of flushes |1
 |`flush.total_time` |`ftt`, `flushTotalTime` |No |Time spent in flush |1
 |`get.current` |`gc`, `getCurrent` |No |Number of current get
diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc
index bcef61d4ef7..8a2f27112b5 100644
--- a/docs/reference/cluster/nodes-stats.asciidoc
+++ b/docs/reference/cluster/nodes-stats.asciidoc
@@ -60,6 +60,9 @@ of `indices`, `os`, `process`, `jvm`, `transport`, `http`,
 `discovery`::
 	Statistics about the discovery
 
+`ingest`::
+    Statistics about ingest preprocessing
+
 [source,js]
 --------------------------------------------------
 # return indices and os
@@ -227,3 +230,23 @@ curl -XGET 'http://localhost:9200/_nodes/stats?pretty&groups=_all'
 # Some groups from just the indices stats
 curl -XGET 'http://localhost:9200/_nodes/stats/indices?pretty&groups=foo,bar'
 --------------------------------------------------
+
+[float]
+[[ingest-stats]]
+=== Ingest statistics
+
+The `ingest` flag can be set to retrieve statistics that concern ingest:
+
+`ingest.total.count`::
+    The total number of document ingested during the lifetime of this node
+
+`ingest.total.time_in_millis`::
+    The total time spent on ingest preprocessing documents during the lifetime of this node
+
+`ingest.total.current`::
+    The total number of documents currently being ingested.
+
+`ingest.total.failed`::
+    The total number ingest preprocessing operations failed during the lifetime of this node
+
+On top of these overall ingest statistics, these statistics are also provided on a per pipeline basis.
\ No newline at end of file
diff --git a/docs/reference/cluster/update-settings.asciidoc b/docs/reference/cluster/update-settings.asciidoc
index 8ec58424730..d0b127cd352 100644
--- a/docs/reference/cluster/update-settings.asciidoc
+++ b/docs/reference/cluster/update-settings.asciidoc
@@ -83,6 +83,16 @@ Cluster wide settings can be returned using:
 curl -XGET localhost:9200/_cluster/settings
 --------------------------------------------------
 
+[float]
+=== Precedence of settings
+
+Transient cluster settings take precedence over persistent cluster settings,
+which take precedence over settings configured in the `elasticsearch.yml`
+config file.
+
+For this reason it is preferrable to  use the `elasticsearch.yml` file only
+for local configurations, and set all cluster-wider settings with the
+`settings` API.
 
 A list of dynamically updatable settings can be found in the
 <<modules,Modules>> documentation.
diff --git a/docs/reference/docs.asciidoc b/docs/reference/docs.asciidoc
index a35fa4c4a89..465d2e60c77 100644
--- a/docs/reference/docs.asciidoc
+++ b/docs/reference/docs.asciidoc
@@ -29,10 +29,14 @@ include::docs/delete.asciidoc[]
 
 include::docs/update.asciidoc[]
 
+include::docs/update-by-query.asciidoc[]
+
 include::docs/multi-get.asciidoc[]
 
 include::docs/bulk.asciidoc[]
 
+include::docs/reindex.asciidoc[]
+
 include::docs/termvectors.asciidoc[]
 
 include::docs/multi-termvectors.asciidoc[]
diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index 1b5483d5ee1..5f4641ca187 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -1,5 +1,5 @@
 [[docs-reindex]]
-==== Reindex API
+== Reindex API
 
 `_reindex`'s most basic form just copies documents from one index to another.
 This will copy documents from `twitter` into `new_twitter`:
@@ -299,7 +299,8 @@ POST /_reindex
 === URL Parameters
 
 In addition to the standard parameters like `pretty`, the Reindex API also
-supports `refresh`, `wait_for_completion`, `consistency`, and `timeout`.
+supports `refresh`, `wait_for_completion`, `consistency`, `timeout`, and
+`requests_per_second`.
 
 Sending the `refresh` url parameter will cause all indexes to which the request
 wrote to be refreshed. This is different than the Index API's `refresh`
@@ -317,8 +318,14 @@ request. `timeout` controls how long each write request waits for unavailable
 shards to become available. Both work exactly how they work in the
 {ref}/docs-bulk.html[Bulk API].
 
-`timeout` controls how long each batch waits for the target shard to become
-available. It works exactly how it works in the {ref}/docs-bulk.html[Bulk API].
+`requests_per_second` can be set to any decimal number (1.4, 6, 1000, etc) and
+throttle the number of requests per second that the reindex issues. The
+throttling is done waiting between bulk batches so that it can manipulate the
+scroll timeout. The wait time is the difference between the time it took the
+batch to complete and the time `requests_per_second * requests_in_the_batch`.
+Since the batch isn't broken into multiple bulk requests large batch sizes will
+cause Elasticsearch to create many requests and then wait for a while before
+starting the next set. This is "bursty" instead of "smooth".
 
 [float]
 === Response body
@@ -333,6 +340,8 @@ The JSON response looks like this:
   "created": 123,
   "batches": 1,
   "version_conflicts": 2,
+  "retries": 0,
+  "throttled_millis": 0,
   "failures" : [ ]
 }
 --------------------------------------------------
@@ -357,6 +366,14 @@ The number of scroll responses pulled back by the the reindex.
 
 The number of version conflicts that reindex hit.
 
+`retries`::
+
+The number of retries that the reindex did in response to a full queue.
+
+`throttled_millis`::
+
+Number of milliseconds the request slept to conform to `requests_per_second`.
+
 `failures`::
 
 Array of all indexing failures. If this is non-empty then the request aborted
@@ -403,7 +420,9 @@ The responses looks like:
           "deleted" : 0,
           "batches" : 36,
           "version_conflicts" : 0,
-          "noops" : 0
+          "noops" : 0,
+          "retries": 0,
+          "throttled_millis": 0
         },
         "description" : ""
       } ]
@@ -420,9 +439,7 @@ will finish when their sum is equal to the `total` field.
 
 
 [float]
-=== Examples
-
-==== Change the name of a field
+=== Reindex to change the name of a field
 
 `_reindex` can be used to build a copy of an index with renamed fields. Say you
 create an index containing documents that look like this:
diff --git a/docs/reference/docs/termvectors.asciidoc b/docs/reference/docs/termvectors.asciidoc
index 0e108430f85..34fcaaecda1 100644
--- a/docs/reference/docs/termvectors.asciidoc
+++ b/docs/reference/docs/termvectors.asciidoc
@@ -136,13 +136,13 @@ curl -s -XPUT 'http://localhost:9200/twitter/' -d '{
     "tweet": {
       "properties": {
         "text": {
-          "type": "string",
+          "type": "text",
           "term_vector": "with_positions_offsets_payloads",
           "store" : true,
           "analyzer" : "fulltext_analyzer"
          },
          "fullname": {
-          "type": "string",
+          "type": "text",
           "term_vector": "with_positions_offsets_payloads",
           "analyzer" : "fulltext_analyzer"
         }
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index c3f57deeaae..52667bf79a8 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -1,5 +1,5 @@
 [[docs-update-by-query]]
-==== Update By Query API
+== Update By Query API
 
 The simplest usage of `_update_by_query` just performs an update on every
 document in the index without changing the source. This is useful to
@@ -101,8 +101,8 @@ Just as in {ref}/docs-update.html[Update API] you can set `ctx.op = "noop"` if
 your script decides that it doesn't have to make any changes. That will cause
 `_update_by_query` to omit that document from its updates. Setting `ctx.op` to
 anything else is an error. If you want to delete by a query you can use the
-<<plugins-delete-by-query,Delete by Query Plugin>> instead. Setting any other
-field in `ctx` is an error.
+{plugins}/plugins-delete-by-query.html[Delete by Query plugin] instead. Setting any
+other field in `ctx` is an error.
 
 Note that we stopped specifying `conflicts=proceed`. In this case we want a
 version conflict to abort the process so we can handle the failure.
@@ -169,8 +169,14 @@ request. `timeout` controls how long each write request waits for unavailable
 shards to become available. Both work exactly how they work in the
 {ref}/docs-bulk.html[Bulk API].
 
-`timeout` controls how long each batch waits for the target shard to become
-available. It works exactly how it works in the {ref}/docs-bulk.html[Bulk API].
+`requests_per_second` can be set to any decimal number (1.4, 6, 1000, etc) and
+throttle the number of requests per second that the update by query issues. The
+throttling is done waiting between bulk batches so that it can manipulate the
+scroll timeout. The wait time is the difference between the time it took the
+batch to complete and the time `requests_per_second * requests_in_the_batch`.
+Since the batch isn't broken into multiple bulk requests large batch sizes will
+cause Elasticsearch to create many requests and then wait for a while before
+starting the next set. This is "bursty" instead of "smooth".
 
 [float]
 === Response body
@@ -184,6 +190,8 @@ The JSON response looks like this:
   "updated": 0,
   "batches": 1,
   "version_conflicts": 2,
+  "retries": 0,
+  "throttled_millis": 0,
   "failures" : [ ]
 }
 --------------------------------------------------
@@ -204,6 +212,14 @@ The number of scroll responses pulled back by the the update by query.
 
 The number of version conflicts that the update by query hit.
 
+`retries`::
+
+The number of retries that the update by query did in response to a full queue.
+
+`throttled_millis`::
+
+Number of milliseconds the request slept to conform to `requests_per_second`.
+
 `failures`::
 
 Array of all indexing failures. If this is non-empty then the request aborted
@@ -251,7 +267,9 @@ The responses looks like:
           "deleted" : 0,
           "batches" : 36,
           "version_conflicts" : 0,
-          "noops" : 0
+          "noops" : 0,
+          "retries": 0,
+          "throttled_millis": 0
         },
         "description" : ""
       } ]
@@ -267,11 +285,8 @@ progress by adding the `updated`, `created`, and `deleted` fields. The request
 will finish when their sum is equal to the `total` field.
 
 
-[float]
-=== Examples
-
 [[picking-up-a-new-property]]
-==== Pick up a new property
+=== Pick up a new property
 
 Say you created an index without dynamic mapping, filled it with data, and then
 added a mapping value to pick up more fields from the data:
@@ -284,7 +299,7 @@ PUT test
     "test": {
       "dynamic": false,   <1>
       "properties": {
-        "text": {"type": "string"}
+        "text": {"type": "text"}
       }
     }
   }
@@ -303,8 +318,8 @@ POST test/test?refresh
 PUT test/_mapping/test   <2>
 {
   "properties": {
-    "text": {"type": "string"},
-    "flag": {"type": "string", "analyzer": "keyword"}
+    "text": {"type": "text"},
+    "flag": {"type": "text", "analyzer": "keyword"}
   }
 }
 --------------------------------------------------
diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc
index 634bc23d6ac..316714259e0 100644
--- a/docs/reference/docs/update.asciidoc
+++ b/docs/reference/docs/update.asciidoc
@@ -251,5 +251,15 @@ sure the document doesn't change during the update. You can use the `version`
 parameter to specify that the document should only be updated if its version
 matches the one specified. By setting version type to `force` you can force
 the new version of the document after update (use with care! with `force`
-there is no guarantee the document didn't change).Version types `external` &
-`external_gte` are not supported.
+there is no guarantee the document didn't change).
+
+[NOTE]
+.The update API does not support external versioning
+=====================================================
+
+External versioning (version types `external` & `external_gte`) is not
+supported by the update API as it would result in Elasticsearch version
+numbers being out of sync with the external system.  Use the
+<<docs-index_,`index` API>> instead.
+
+=====================================================
diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc
index 8ff832c673f..47bcb3031ff 100755
--- a/docs/reference/getting-started.asciidoc
+++ b/docs/reference/getting-started.asciidoc
@@ -163,7 +163,7 @@ As mentioned previously, we can override either the cluster or node name. This c
 
 [source,sh]
 --------------------------------------------------
-./elasticsearch --cluster.name my_cluster_name --node.name my_node_name
+./elasticsearch -Ees.cluster.name=my_cluster_name -Ees.node.name=my_node_name
 --------------------------------------------------
 
 Also note the line marked http with information about the HTTP address (`192.168.8.112`) and port (`9200`) that our node is reachable from. By default, Elasticsearch uses port `9200` to provide access to its REST API. This port is configurable if necessary.
diff --git a/docs/reference/index-modules/allocation/filtering.asciidoc b/docs/reference/index-modules/allocation/filtering.asciidoc
index 784fa1af24c..44c9b1a712c 100644
--- a/docs/reference/index-modules/allocation/filtering.asciidoc
+++ b/docs/reference/index-modules/allocation/filtering.asciidoc
@@ -14,7 +14,7 @@ attribute as follows:
 
 [source,sh]
 ------------------------
-bin/elasticsearch --node.rack rack1 --node.size big  <1>
+bin/elasticsearch -Ees.node.rack=rack1 -Ees.node.size=big  <1>
 ------------------------
 <1> These attribute settings can also be specified in the `elasticsearch.yml` config file.
 
diff --git a/docs/reference/index-modules/similarity.asciidoc b/docs/reference/index-modules/similarity.asciidoc
index 3d993a3a6eb..07591dc277b 100644
--- a/docs/reference/index-modules/similarity.asciidoc
+++ b/docs/reference/index-modules/similarity.asciidoc
@@ -39,7 +39,7 @@ Here we configure the DFRSimilarity so it can be referenced as
 {
   "book" : {
     "properties" : {
-      "title" : { "type" : "string", "similarity" : "my_similarity" }
+      "title" : { "type" : "text", "similarity" : "my_similarity" }
     }
 }
 --------------------------------------------------
diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc
index 87c400e018c..abfcb18de2e 100644
--- a/docs/reference/index.asciidoc
+++ b/docs/reference/index.asciidoc
@@ -47,9 +47,7 @@ include::testing.asciidoc[]
 
 include::glossary.asciidoc[]
 
-//////////////////////////////////////////
-  include::release-notes.asciidoc[]
-//////////////////////////////////////////
+include::release-notes.asciidoc[]
 
 include::redirects.asciidoc[]
 
diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc
index cb8f652070b..ab4e49e7aab 100644
--- a/docs/reference/indices/aliases.asciidoc
+++ b/docs/reference/indices/aliases.asciidoc
@@ -116,8 +116,7 @@ curl -XPUT 'http://localhost:9200/test1' -d '{
     "type1": {
       "properties": {
         "user" : {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         }
       }
     }
diff --git a/docs/reference/indices/clearcache.asciidoc b/docs/reference/indices/clearcache.asciidoc
index 21008e5b46b..8ebb9e3488a 100644
--- a/docs/reference/indices/clearcache.asciidoc
+++ b/docs/reference/indices/clearcache.asciidoc
@@ -2,7 +2,7 @@
 == Clear Cache
 
 The clear cache API allows to clear either all caches or specific cached
-associated with one ore more indices.
+associated with one or more indices.
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/indices/create-index.asciidoc b/docs/reference/indices/create-index.asciidoc
index 2210870135b..11216fa4c01 100644
--- a/docs/reference/indices/create-index.asciidoc
+++ b/docs/reference/indices/create-index.asciidoc
@@ -78,7 +78,7 @@ curl -XPOST localhost:9200/test -d '{
     "mappings" : {
         "type1" : {
             "properties" : {
-                "field1" : { "type" : "string", "index" : "not_analyzed" }
+                "field1" : { "type" : "text" }
             }
         }
     }
diff --git a/docs/reference/indices/get-field-mapping.asciidoc b/docs/reference/indices/get-field-mapping.asciidoc
index 2aeb853e9f4..39667dc0874 100644
--- a/docs/reference/indices/get-field-mapping.asciidoc
+++ b/docs/reference/indices/get-field-mapping.asciidoc
@@ -22,7 +22,7 @@ For which the response is (assuming `text` is a default string field):
          "text": {
             "full_name": "text",
             "mapping": {
-               "text": { "type": "string" }
+               "text": { "type": "text" }
             }
          }
       }
@@ -73,13 +73,13 @@ For example, consider the following mapping:
  {
      "article": {
          "properties": {
-             "id": { "type": "string" },
-             "title":  { "type": "string"},
-             "abstract": { "type": "string"},
+             "id": { "type": "text" },
+             "title":  { "type": "text"},
+             "abstract": { "type": "text"},
              "author": {
                  "properties": {
-                     "id": { "type": "string" },
-                     "name": { "type": "string" }
+                     "id": { "type": "text" },
+                     "name": { "type": "text" }
                  }
              }
          }
@@ -105,19 +105,19 @@ returns:
          "abstract": {
             "full_name": "abstract",
             "mapping": {
-               "abstract": { "type": "string" }
+               "abstract": { "type": "text" }
             }
          },
          "author.id": {
             "full_name": "author.id",
             "mapping": {
-               "id": { "type": "string" }
+               "id": { "type": "text" }
             }
          },
          "name": {
             "full_name": "author.name",
             "mapping": {
-               "name": { "type": "string" }
+               "name": { "type": "text" }
             }
          }
       }
diff --git a/docs/reference/indices/put-mapping.asciidoc b/docs/reference/indices/put-mapping.asciidoc
index 7dd2389e824..cc94a08f626 100644
--- a/docs/reference/indices/put-mapping.asciidoc
+++ b/docs/reference/indices/put-mapping.asciidoc
@@ -12,7 +12,7 @@ PUT twitter <1>
     "tweet": {
       "properties": {
         "message": {
-          "type": "string"
+          "type": "text"
         }
       }
     }
@@ -23,7 +23,7 @@ PUT twitter/_mapping/user <2>
 {
   "properties": {
     "name": {
-      "type": "string"
+      "type": "text"
     }
   }
 }
@@ -32,7 +32,7 @@ PUT twitter/_mapping/tweet <3>
 {
   "properties": {
     "user_name": {
-      "type": "string"
+      "type": "text"
     }
   }
 }
@@ -86,13 +86,12 @@ PUT my_index <1>
         "name": {
           "properties": {
             "first": {
-              "type": "string"
+              "type": "text"
             }
           }
         },
         "user_id": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         }
       }
     }
@@ -105,13 +104,12 @@ PUT my_index/_mapping/user
     "name": {
       "properties": {
         "last": { <2>
-          "type": "string"
+          "type": "text"
         }
       }
     },
     "user_id": {
-      "type": "string",
-      "index": "not_analyzed",
+      "type": "keyword",
       "ignore_above": 100 <3>
     }
   }
@@ -149,7 +147,7 @@ PUT my_index
     "type_one": {
       "properties": {
         "text": { <1>
-          "type": "string",
+          "type": "text",
           "analyzer": "standard"
         }
       }
@@ -157,7 +155,7 @@ PUT my_index
     "type_two": {
       "properties": {
         "text": { <1>
-          "type": "string",
+          "type": "text",
           "analyzer": "standard"
         }
       }
@@ -169,7 +167,7 @@ PUT my_index/_mapping/type_one <2>
 {
   "properties": {
     "text": {
-      "type": "string",
+      "type": "text",
       "analyzer": "standard",
       "search_analyzer": "whitespace"
     }
@@ -180,7 +178,7 @@ PUT my_index/_mapping/type_one?update_all_types <3>
 {
   "properties": {
     "text": {
-      "type": "string",
+      "type": "text",
       "analyzer": "standard",
       "search_analyzer": "whitespace"
     }
diff --git a/docs/reference/indices/stats.asciidoc b/docs/reference/indices/stats.asciidoc
index 87eda44e104..e990a7ff6bd 100644
--- a/docs/reference/indices/stats.asciidoc
+++ b/docs/reference/indices/stats.asciidoc
@@ -34,7 +34,8 @@ specified as well in the URI. Those stats can be any of:
 
 `get`:: 		Get statistics, including missing stats.
 
-`search`:: 		Search statistics. You can include statistics for custom groups by adding
+`search`:: 		Search statistics including suggest statistics.
+                You can include statistics for custom groups by adding
                 an extra `groups` parameter (search operations can be associated with one or more
                 groups). The `groups` parameter accepts a comma separated list of group names.
                 Use `_all` to return statistics for all groups.
@@ -47,7 +48,6 @@ specified as well in the URI. Those stats can be any of:
 `merge`::       Merge statistics.
 `request_cache`:: <<shard-request-cache,Shard request cache>> statistics.
 `refresh`::     Refresh statistics.
-`suggest`::     Suggest statistics.
 `warmer`::      Warmer statistics.
 `translog`::    Translog statistics.
 
diff --git a/docs/reference/ingest.asciidoc b/docs/reference/ingest.asciidoc
index b2486c7f12b..c565f3b2047 100644
--- a/docs/reference/ingest.asciidoc
+++ b/docs/reference/ingest.asciidoc
@@ -15,10 +15,10 @@ on all nodes. To disable ingest on a node, configure the following setting in th
 node.ingest: false
 --------------------------------------------------
 
-To pre-process documents before indexing, you <<pipe-line,define a pipeline>> that specifies
+To pre-process documents before indexing, you <<pipeline,define a pipeline>> that specifies
 a series of <<ingest-processors,processors>>. Each processor transforms the document in some way.
 For example, you may have a pipeline that consists of one processor that removes a field from
-the document followed by another processor that renames a field. 
+the document followed by another processor that renames a field.
 
 To use a pipeline, you simply specify the `pipeline` parameter on an index or bulk request to
 tell the ingest node which pipeline to use. For example:
@@ -32,7 +32,7 @@ PUT /my-index/my-type/my-id?pipeline=my_pipeline_id
 --------------------------------------------------
 // AUTOSENSE
 
-See <<ingest-apis,Ingest APIs>> for more information about creating, adding, and deleting pipelines. 
+See <<ingest-apis,Ingest APIs>> for more information about creating, adding, and deleting pipelines.
 
 --
 
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index 0827baa6ea1..10b640dbaf1 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -1,4 +1,4 @@
-[[pipe-line]]
+[[pipeline]]
 == Pipeline Definition
 
 A pipeline is a definition of  a series of <<ingest-processors, processors>> that are to be executed
@@ -634,6 +634,9 @@ plugin.mandatory: ingest-attachment,ingest-geoip
 
 A node will not start if either of these plugins are not available.
 
+The <<ingest-stats,node stats API>> can be used to fetch ingest usage statistics, globally and on a per
+pipeline basis. Useful to find out which pipelines are used the most or spent the most time on preprocessing.
+
 [[append-procesesor]]
 === Append Processor
 Appends one or more values to an existing array if the field already exists and it is an array.
diff --git a/docs/reference/mapping.asciidoc b/docs/reference/mapping.asciidoc
index 407f43625fa..8ead0436978 100644
--- a/docs/reference/mapping.asciidoc
+++ b/docs/reference/mapping.asciidoc
@@ -46,7 +46,7 @@ Fields with the same name in different mapping types in the same index
 
 Each field has a data `type` which can be:
 
-* a simple type like <<string,`string`>>, <<date,`date`>>, <<number,`long`>>,
+* a simple type like <<text,`text`>>, <<keyword,`keyword`>>, <<date,`date`>>, <<number,`long`>>,
   <<number,`double`>>, <<boolean,`boolean`>> or <<ip,`ip`>>.
 * a type which supports the hierarchical nature of JSON such as
   <<object,`object`>> or <<nested,`nested`>>.
@@ -55,7 +55,7 @@ Each field has a data `type` which can be:
 
 It is often useful to index the same field in different ways for different
 purposes. For instance, a `string` field could be <<mapping-index,indexed>> as
-an `analyzed` field for full-text search, and as a `not_analyzed` field for
+a `text` field for full-text search, and as a `keyword` field for
 sorting or aggregations.  Alternatively, you could index a string field with
 the <<analysis-standard-analyzer,`standard` analyzer>>, the
 <<english-analyzer,`english`>> analyzer, and the
@@ -134,18 +134,17 @@ PUT my_index <1>
     "user": { <2>
       "_all":       { "enabled": false  }, <3>
       "properties": { <4>
-        "title":    { "type": "string"  }, <5>
-        "name":     { "type": "string"  }, <5>
+        "title":    { "type": "text"  }, <5>
+        "name":     { "type": "text"  }, <5>
         "age":      { "type": "integer" }  <5>
       }
     },
     "blogpost": { <2>
       "properties": { <4>
-        "title":    { "type": "string"  }, <5>
-        "body":     { "type": "string"  }, <5>
+        "title":    { "type": "text"  }, <5>
+        "body":     { "type": "text"  }, <5>
         "user_id":  {
-          "type":   "string", <5>
-          "index":  "not_analyzed"
+          "type":   "keyword" <5>
         },
         "created":  {
           "type":   "date", <5>
diff --git a/docs/reference/mapping/dynamic/default-mapping.asciidoc b/docs/reference/mapping/dynamic/default-mapping.asciidoc
index c1e1f8dec66..bef90301f0c 100644
--- a/docs/reference/mapping/dynamic/default-mapping.asciidoc
+++ b/docs/reference/mapping/dynamic/default-mapping.asciidoc
@@ -56,11 +56,10 @@ PUT _template/logging
           "strings": { <4>
             "match_mapping_type": "string",
             "mapping": {
-              "type": "string",
+              "type": "text",
               "fields": {
                 "raw": {
-                  "type":  "string",
-                  "index": "not_analyzed",
+                  "type":  "keyword",
                   "ignore_above": 256
                 }
               }
@@ -79,4 +78,4 @@ PUT logs-2015.10.01/event/1
 <1> The `logging` template will match any indices beginning with `logs-`.
 <2> Matching indices will be created with a single primary shard.
 <3> The `_all` field will be disabled by default for new type mappings.
-<4> String fields will be created with an `analyzed` main field, and a `not_analyzed` `.raw` field.
+<4> String fields will be created with a `text` main field, and a `keyword` `.raw` field.
diff --git a/docs/reference/mapping/dynamic/field-mapping.asciidoc b/docs/reference/mapping/dynamic/field-mapping.asciidoc
index 585931d5e3f..f8612958f9c 100644
--- a/docs/reference/mapping/dynamic/field-mapping.asciidoc
+++ b/docs/reference/mapping/dynamic/field-mapping.asciidoc
@@ -22,7 +22,7 @@ string::                            Either a <<date,`date`>> field
                                         (if the value passes <<date-detection,date detection>>),
                                     a <<number,`double`>> or <<number,`long`>> field
                                         (if the value passes <<numeric-detection,numeric detection>>)
-                                    or an <<mapping-index,`analyzed`>> <<string,`string`>> field.
+                                    or an <<text,`text`>> field.
 
 These are the only <<mapping-types,field datatypes>> that are dynamically
 detected.  All other datatypes must be mapped explicitly.
@@ -81,7 +81,7 @@ PUT my_index/my_type/1 <1>
 --------------------------------------------------
 // AUTOSENSE
 
-<1> The `create_date` field has been added as a <<string,`string`>> field.
+<1> The `create_date` field has been added as a <<text,`text`>> field.
 
 ===== Customising detected date formats
 
diff --git a/docs/reference/mapping/dynamic/templates.asciidoc b/docs/reference/mapping/dynamic/templates.asciidoc
index b903f1af066..1137f454ffd 100644
--- a/docs/reference/mapping/dynamic/templates.asciidoc
+++ b/docs/reference/mapping/dynamic/templates.asciidoc
@@ -52,7 +52,7 @@ can be automatically detected: `boolean`, `date`, `double`, `long`, `object`,
 `string`.  It also accepts `*` to match all datatypes.
 
 For example, if we wanted to map all integer fields as `integer` instead of
-`long`, and all `string` fields as both `analyzed` and `not_analyzed`, we
+`long`, and all `string` fields as both `text` and `keyword`, we
 could use the following template:
 
 [source,js]
@@ -74,11 +74,10 @@ PUT my_index
           "strings": {
             "match_mapping_type": "string",
             "mapping": {
-              "type": "string",
+              "type": "text",
               "fields": {
                 "raw": {
-                  "type":  "string",
-                  "index": "not_analyzed",
+                  "type":  "keyword",
                   "ignore_above": 256
                 }
               }
@@ -99,7 +98,7 @@ PUT my_index/my_type/1
 --------------------------------------------------
 // AUTOSENSE
 <1> The `my_integer` field is mapped as an `integer`.
-<2> The `my_string` field is mapped as an analyzed `string`, with a `not_analyzed` <<multi-fields,multi field>>.
+<2> The `my_string` field is mapped as a `text`, with a `keyword` <<multi-fields,multi field>>.
 
 
 [[match-unmatch]]
@@ -180,7 +179,7 @@ PUT my_index
             "path_match":   "name.*",
             "path_unmatch": "*.middle",
             "mapping": {
-              "type":       "string",
+              "type":       "text",
               "copy_to":    "full_name"
             }
           }
@@ -221,7 +220,7 @@ PUT my_index
             "match_mapping_type": "string",
             "match": "*",
             "mapping": {
-              "type": "string",
+              "type": "text",
               "analyzer": "{name}"
             }
           }
diff --git a/docs/reference/mapping/fields/all-field.asciidoc b/docs/reference/mapping/fields/all-field.asciidoc
index ae52fc1d0d1..6c5f073aee8 100644
--- a/docs/reference/mapping/fields/all-field.asciidoc
+++ b/docs/reference/mapping/fields/all-field.asciidoc
@@ -45,7 +45,7 @@ from each field as a string. It does not combine the _terms_ from each field.
 
 =============================================================================
 
-The `_all` field is just a <<string,`string`>> field, and accepts the same
+The `_all` field is just a <<text,`text`>> field, and accepts the same
 parameters that  other string fields accept, including `analyzer`,
 `term_vectors`, `index_options`, and `store`.
 
@@ -136,7 +136,7 @@ PUT my_index
       },
       "properties": {
         "content": {
-          "type": "string"
+          "type": "text"
         }
       }
     }
@@ -172,11 +172,11 @@ PUT myindex
     "mytype": {
       "properties": {
         "title": { <1>
-          "type": "string",
+          "type": "text",
           "boost": 2
         },
         "content": { <1>
-          "type": "string"
+          "type": "text"
         }
       }
     }
@@ -210,15 +210,15 @@ PUT myindex
     "mytype": {
       "properties": {
         "first_name": {
-          "type":    "string",
+          "type":    "text",
           "copy_to": "full_name" <1>
         },
         "last_name": {
-          "type":    "string",
+          "type":    "text",
           "copy_to": "full_name" <1>
         },
         "full_name": {
-          "type":    "string"
+          "type":    "text"
         }
       }
     }
diff --git a/docs/reference/mapping/fields/parent-field.asciidoc b/docs/reference/mapping/fields/parent-field.asciidoc
index 64f4a9934a6..fb066580044 100644
--- a/docs/reference/mapping/fields/parent-field.asciidoc
+++ b/docs/reference/mapping/fields/parent-field.asciidoc
@@ -127,7 +127,7 @@ global ordinals for the `_parent` field.
 Global ordinals, by default, are built lazily: the first parent-child query or
 aggregation after a refresh will trigger building of global ordinals. This can
 introduce a significant latency spike for your users. You can use
-<<fielddata-loading,eager_global_ordinals>> to shift the cost of building global
+<<global-ordinals,eager_global_ordinals>> to shift the cost of building global
 ordinals from query time to refresh time, by mapping the `_parent` field as follows:
 
 [source,js]
@@ -139,9 +139,7 @@ PUT my_index
     "my_child": {
       "_parent": {
         "type": "my_parent",
-        "fielddata": {
-          "loading": "eager_global_ordinals"
-        }
+        "eager_global_ordinals": true
       }
     }
   }
diff --git a/docs/reference/mapping/params/analyzer.asciidoc b/docs/reference/mapping/params/analyzer.asciidoc
index 68009e600db..2a452465e2d 100644
--- a/docs/reference/mapping/params/analyzer.asciidoc
+++ b/docs/reference/mapping/params/analyzer.asciidoc
@@ -47,10 +47,10 @@ PUT my_index
     "my_type": {
       "properties": {
         "text": { <1>
-          "type": "string",
+          "type": "text",
           "fields": {
             "english": { <2>
-              "type":     "string",
+              "type":     "text",
               "analyzer": "english"
             }
           }
@@ -124,7 +124,7 @@ PUT /my_index
       "my_type":{
          "properties":{
             "title": {
-               "type":"string",
+               "type":"text",
                "analyzer":"my_analyzer", <3>
                "search_analyzer":"my_stop_analyzer", <4>
                "search_quote_analyzer":"my_analyzer" <5>
diff --git a/docs/reference/mapping/params/boost.asciidoc b/docs/reference/mapping/params/boost.asciidoc
index 22c0e2e69ea..add6f806844 100644
--- a/docs/reference/mapping/params/boost.asciidoc
+++ b/docs/reference/mapping/params/boost.asciidoc
@@ -12,11 +12,11 @@ PUT my_index
     "my_type": {
       "properties": {
         "title": {
-          "type": "string",
+          "type": "text",
           "boost": 2 <1>
         },
         "content": {
-          "type": "string"
+          "type": "text"
         }
       }
     }
@@ -83,4 +83,4 @@ We advise against using index time boosting for the following reasons:
   byte.  This reduces the resolution of the field length normalization factor
   which can lead to lower quality relevance calculations.
 
-==================================================
\ No newline at end of file
+==================================================
diff --git a/docs/reference/mapping/params/coerce.asciidoc b/docs/reference/mapping/params/coerce.asciidoc
index c9491607a6b..0121c307230 100644
--- a/docs/reference/mapping/params/coerce.asciidoc
+++ b/docs/reference/mapping/params/coerce.asciidoc
@@ -12,7 +12,6 @@ For instance:
 
 * Strings will be coerced to numbers.
 * Floating points will be truncated for integer values.
-* Lon/lat geo-points will be normalized to a standard -180:180 / -90:90 coordinate system.
 
 For instance:
 
diff --git a/docs/reference/mapping/params/copy-to.asciidoc b/docs/reference/mapping/params/copy-to.asciidoc
index b437a87424a..863bf1996cd 100644
--- a/docs/reference/mapping/params/copy-to.asciidoc
+++ b/docs/reference/mapping/params/copy-to.asciidoc
@@ -15,15 +15,15 @@ PUT /my_index
     "my_type": {
       "properties": {
         "first_name": {
-          "type": "string",
+          "type": "text",
           "copy_to": "full_name" <1>
         },
         "last_name": {
-          "type": "string",
+          "type": "text",
           "copy_to": "full_name" <1>
         },
         "full_name": {
-          "type": "string"
+          "type": "text"
         }
       }
     }
diff --git a/docs/reference/mapping/params/doc-values.asciidoc b/docs/reference/mapping/params/doc-values.asciidoc
index 81f9b6e3c64..4ded2212de1 100644
--- a/docs/reference/mapping/params/doc-values.asciidoc
+++ b/docs/reference/mapping/params/doc-values.asciidoc
@@ -29,12 +29,10 @@ PUT my_index
     "my_type": {
       "properties": {
         "status_code": { <1>
-          "type":       "string",
-          "index":      "not_analyzed"
+          "type":       "keyword"
         },
         "session_id": { <2>
-          "type":       "string",
-          "index":      "not_analyzed",
+          "type":       "keyword",
           "doc_values": false
         }
       }
diff --git a/docs/reference/mapping/params/dynamic.asciidoc b/docs/reference/mapping/params/dynamic.asciidoc
index db73709f4f3..72bbd369d7f 100644
--- a/docs/reference/mapping/params/dynamic.asciidoc
+++ b/docs/reference/mapping/params/dynamic.asciidoc
@@ -67,7 +67,7 @@ PUT my_index
         "user": { <2>
           "properties": {
             "name": {
-              "type": "string"
+              "type": "text"
             },
             "social_networks": { <3>
               "dynamic": true,
diff --git a/docs/reference/mapping/params/enabled.asciidoc b/docs/reference/mapping/params/enabled.asciidoc
index 6f72f4da890..7bffcfddf2e 100644
--- a/docs/reference/mapping/params/enabled.asciidoc
+++ b/docs/reference/mapping/params/enabled.asciidoc
@@ -21,8 +21,7 @@ PUT my_index
     "session": {
       "properties": {
         "user_id": {
-          "type":  "string",
-          "index": "not_analyzed"
+          "type":  "keyword"
         },
         "last_updated": {
           "type": "date"
diff --git a/docs/reference/mapping/params/fielddata.asciidoc b/docs/reference/mapping/params/fielddata.asciidoc
index 4d96fb61132..e67b47a831f 100644
--- a/docs/reference/mapping/params/fielddata.asciidoc
+++ b/docs/reference/mapping/params/fielddata.asciidoc
@@ -12,28 +12,28 @@ documents, we need to be able to look up the document and find the terms that
 it has in a field.
 
 Most fields can use index-time, on-disk <<doc-values,`doc_values`>> to support
-this type of data access pattern, but `analyzed` string fields do not support
-`doc_values`.
+this type of data access pattern, but `text` fields do not support `doc_values`.
 
-Instead, `analyzed` strings use a query-time data structure called
+Instead, `text` strings use a query-time data structure called
 `fielddata`.  This data structure is built on demand the first time that a
 field is used for aggregations, sorting, or is accessed in a script.  It is built
 by reading the entire inverted index for each segment from disk, inverting the
 term ↔︎ document relationship, and storing the result in memory, in the
 JVM heap.
 
-Loading fielddata is an expensive process so, once it has been loaded, it
-remains in memory for the lifetime of the segment.
+Loading fielddata is an expensive process so it is disabled by default. Also,
+when enabled, once it has been loaded, it remains in memory for the lifetime of
+the segment.
 
 [WARNING]
 .Fielddata can fill up your heap space
 ==============================================================================
 Fielddata can consume a lot of heap space, especially when loading high
-cardinality `analyzed` string fields.  Most of the time, it doesn't make sense
-to sort or aggregate on `analyzed` string fields (with the notable exception
+cardinality `text` fields.  Most of the time, it doesn't make sense
+to sort or aggregate on `text` fields (with the notable exception
 of the
 <<search-aggregations-bucket-significantterms-aggregation,`significant_terms`>>
-aggregation).  Always think about whether a `not_analyzed` field (which can
+aggregation).  Always think about whether a <<keyword,`keyword`>> field (which can
 use `doc_values`) would be  a better fit for your use case.
 ==============================================================================
 
@@ -42,71 +42,6 @@ same name in the same index.  Its value can be updated on existing fields
 using the <<indices-put-mapping,PUT mapping API>>.
 
 
-[[fielddata-format]]
-==== `fielddata.format`
-
-For `analyzed` string fields, the fielddata `format` controls whether
-fielddata should be enabled or not.  It accepts: `disabled` and `paged_bytes`
-(enabled, which is the default).  To disable fielddata loading, you can use
-the following mapping:
-
-[source,js]
---------------------------------------------------
-PUT my_index
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "text": {
-          "type": "string",
-          "fielddata": {
-            "format": "disabled" <1>
-          }
-        }
-      }
-    }
-  }
-}
---------------------------------------------------
-// AUTOSENSE
-<1> The `text` field cannot be used for sorting, aggregations, or in scripts.
-
-.Fielddata and other datatypes
-[NOTE]
-==================================================
-
-Historically, other field datatypes also used fielddata, but this has been replaced
-by index-time, disk-based <<doc-values,`doc_values`>>.
-
-==================================================
-
-
-[[fielddata-loading]]
-==== `fielddata.loading`
-
-This per-field setting controls when fielddata is loaded into memory. It
-accepts three options:
-
-[horizontal]
-`lazy`::
-
-    Fielddata is only loaded into memory when it is needed. (default)
-
-`eager`::
-
-    Fielddata is loaded into memory before a new search segment becomes
-    visible to search.  This can reduce the latency that a user may experience
-    if their search request has to trigger lazy loading from a big segment.
-
-`eager_global_ordinals`::
-
-    Loading fielddata into memory is only part of the work that is required.
-    After loading the fielddata for each segment, Elasticsearch builds the
-    <<global-ordinals>> data structure to make a list of all unique terms
-    across all the segments in a shard.  By default, global ordinals are built
-    lazily.  If the field has a very high cardinality, global ordinals may
-    take some time to build, in which case you can use eager loading instead.
-
 [[global-ordinals]]
 .Global ordinals
 *****************************************
@@ -141,15 +76,10 @@ can move the loading time from the first search request, to the refresh itself.
 *****************************************
 
 [[field-data-filtering]]
-==== `fielddata.filter`
+==== `fielddata_frequency_filter`
 
 Fielddata filtering can be used to reduce the number of terms loaded into
-memory, and thus reduce memory usage. Terms can be filtered by _frequency_ or
-by _regular expression_, or a combination of the two:
-
-Filtering by frequency::
-+
---
+memory, and thus reduce memory usage. Terms can be filtered by _frequency_:
 
 The frequency filter allows you to only load terms whose term frequency falls
 between a `min` and `max` value, which can be expressed an absolute
@@ -169,7 +99,7 @@ PUT my_index
     "my_type": {
       "properties": {
         "tag": {
-          "type": "string",
+          "type": "text",
           "fielddata": {
             "filter": {
               "frequency": {
@@ -186,44 +116,3 @@ PUT my_index
 }
 --------------------------------------------------
 // AUTOSENSE
---
-
-Filtering by regex::
-+
---
-Terms can also be filtered by regular expression - only values which
-match the regular expression are loaded. Note: the regular expression is
-applied to each term in the field, not to the whole field value. For
-instance, to only load hashtags from a tweet, we can use a regular
-expression which matches terms beginning with `#`:
-
-[source,js]
---------------------------------------------------
-PUT my_index
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "tweet": {
-          "type": "string",
-          "analyzer": "whitespace",
-          "fielddata": {
-            "filter": {
-              "regex": {
-                "pattern": "^#.*"
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-}
---------------------------------------------------
-// AUTOSENSE
---
-
-These filters can be updated on an existing field mapping and will take
-effect the next time the fielddata for a segment is loaded. Use the
-<<indices-clearcache,Clear Cache>> API
-to reload the fielddata using the new filters.
diff --git a/docs/reference/mapping/params/ignore-above.asciidoc b/docs/reference/mapping/params/ignore-above.asciidoc
index 3a8e527860a..a9fa5377144 100644
--- a/docs/reference/mapping/params/ignore-above.asciidoc
+++ b/docs/reference/mapping/params/ignore-above.asciidoc
@@ -1,12 +1,7 @@
 [[ignore-above]]
 === `ignore_above`
 
-Strings longer than the `ignore_above` setting will not be processed by the
-<<analyzer,analyzer>> and will not be indexed. This is mainly useful for
-<<mapping-index,`not_analyzed`>> string fields, which are typically used for
-filtering, aggregations, and sorting.  These are structured fields and it
-doesn't usually make sense to allow very long terms to be indexed in these
-fields.
+Strings longer than the `ignore_above` setting will not be indexed or stored.
 
 [source,js]
 --------------------------------------------------
@@ -16,8 +11,7 @@ PUT my_index
     "my_type": {
       "properties": {
         "message": {
-          "type": "string",
-          "index": "not_analyzed",
+          "type": "keyword",
           "ignore_above": 20 <1>
         }
       }
diff --git a/docs/reference/mapping/params/include-in-all.asciidoc b/docs/reference/mapping/params/include-in-all.asciidoc
index 5061c16d374..76a0d14527c 100644
--- a/docs/reference/mapping/params/include-in-all.asciidoc
+++ b/docs/reference/mapping/params/include-in-all.asciidoc
@@ -14,10 +14,10 @@ PUT my_index
     "my_type": {
       "properties": {
         "title": { <1>
-          "type": "string"
+          "type": "text"
         }
         "content": { <1>
-          "type": "string"
+          "type": "text"
         },
         "date": { <2>
           "type": "date",
@@ -50,18 +50,18 @@ PUT my_index
     "my_type": {
       "include_in_all": false, <1>
       "properties": {
-        "title":          { "type": "string" },
+        "title":          { "type": "text" },
         "author": {
           "include_in_all": true, <2>
           "properties": {
-            "first_name": { "type": "string" },
-            "last_name":  { "type": "string" }
+            "first_name": { "type": "text" },
+            "last_name":  { "type": "text" }
           }
         },
         "editor": {
           "properties": {
-            "first_name": { "type": "string" }, <3>
-            "last_name":  { "type": "string", "include_in_all": true } <3>
+            "first_name": { "type": "text" }, <3>
+            "last_name":  { "type": "text", "include_in_all": true } <3>
           }
         }
       }
diff --git a/docs/reference/mapping/params/index-options.asciidoc b/docs/reference/mapping/params/index-options.asciidoc
index f4608714258..9f327adb969 100644
--- a/docs/reference/mapping/params/index-options.asciidoc
+++ b/docs/reference/mapping/params/index-options.asciidoc
@@ -39,7 +39,7 @@ PUT my_index
     "my_type": {
       "properties": {
         "text": {
-          "type": "string",
+          "type": "text",
           "index_options": "offsets"
         }
       }
diff --git a/docs/reference/mapping/params/index.asciidoc b/docs/reference/mapping/params/index.asciidoc
index 6dd9151814d..e097293d142 100644
--- a/docs/reference/mapping/params/index.asciidoc
+++ b/docs/reference/mapping/params/index.asciidoc
@@ -1,48 +1,6 @@
 [[mapping-index]]
 === `index`
 
-The `index` option controls how field values are indexed and, thus, how they
-are searchable.  It accepts three values:
+The `index` option controls whether field values are indexed. It accepts `true`
+or `false`. Fields that are not indexed are not queryable.
 
-[horizontal]
-`no`::
-
-    Do not add this field value to the index. With this setting, the field
-    will not be queryable.
-
-`not_analyzed`::
-
-    Add the field value to the index unchanged, as a single term.  This is the
-    default for all fields that support this option except for
-    <<string,`string`>> fields.  `not_analyzed` fields are usually used with
-    <<term-level-queries,term-level queries>> for structured search.
-
-`analyzed`::
-
-    This option applies only to `string` fields, for which it is the default.
-    The string field value is first <<analysis,analyzed>> to convert the
-    string into terms (e.g. a list of individual words), which are then
-    indexed.  At search time, the query string is passed through
-    (<<search-analyzer,usually>>) the same analyzer to generate terms
-    in the same format as those in the index.  It is this process that enables
-    <<full-text-queries,full text search>>.
-
-For example, you can create a `not_analyzed` string field with the following:
-
-[source,js]
---------------------------------------------------
-PUT /my_index
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "status_code": {
-          "type": "string",
-          "index": "not_analyzed"
-        }
-      }
-    }
-  }
-}
---------------------------------------------------
-// AUTOSENSE
\ No newline at end of file
diff --git a/docs/reference/mapping/params/multi-fields.asciidoc b/docs/reference/mapping/params/multi-fields.asciidoc
index 994d2fddbc1..8ca2809c7f2 100644
--- a/docs/reference/mapping/params/multi-fields.asciidoc
+++ b/docs/reference/mapping/params/multi-fields.asciidoc
@@ -3,8 +3,8 @@
 
 It is often useful to index the same field in different ways for different
 purposes.  This is the purpose of _multi-fields_. For instance, a `string`
-field could be <<mapping-index,indexed>> as an `analyzed` field for full-text
-search, and as a `not_analyzed` field for sorting or aggregations:
+field could be mapped as a `text` field for full-text
+search, and as a `keyword` field for sorting or aggregations:
 
 [source,js]
 --------------------------------------------------
@@ -14,11 +14,10 @@ PUT /my_index
     "my_type": {
       "properties": {
         "city": {
-          "type": "string",
+          "type": "text",
           "fields": {
             "raw": { <1>
-              "type":  "string",
-              "index": "not_analyzed"
+              "type":  "keyword"
             }
           }
         }
@@ -57,8 +56,8 @@ GET /my_index/_search
 }
 --------------------------------------------------
 // AUTOSENSE
-<1> The `city.raw` field is a `not_analyzed` version of the `city` field.
-<2> The analyzed `city` field can be used for full text search.
+<1> The `city.raw` field is a `keyword` version of the `city` field.
+<2> The `city` field can be used for full text search.
 <3> The `city.raw` field can be used for sorting and aggregations
 
 NOTE: Multi-fields do not change the original `_source` field.
@@ -83,10 +82,10 @@ PUT my_index
     "my_type": {
       "properties": {
         "text": { <1>
-          "type": "string",
+          "type": "text",
           "fields": {
             "english": { <2>
-              "type":     "string",
+              "type":     "text",
               "analyzer": "english"
             }
           }
diff --git a/docs/reference/mapping/params/norms.asciidoc b/docs/reference/mapping/params/norms.asciidoc
index f83f93caf56..f6e42219a1f 100644
--- a/docs/reference/mapping/params/norms.asciidoc
+++ b/docs/reference/mapping/params/norms.asciidoc
@@ -4,14 +4,14 @@
 Norms store various normalization factors that are later used at query time
 in order to compute the score of a document relatively to a query.
 
-Although useful for scoring, norms also require quite a lot of memory
+Although useful for scoring, norms also require quite a lot of disk
 (typically in the order of one byte per document per field in your index, even
 for documents that don't have this specific field). As a consequence, if you
 don't need scoring on a specific field, you should disable norms on that
 field. In  particular, this is the case for fields that are used solely for
 filtering or aggregations.
 
-TIP: The `norms.enabled` setting must have the same setting for fields of the
+TIP: The `norms` setting must have the same setting for fields of the
 same name in the same index.  Norms can be disabled on existing fields using
 the <<indices-put-mapping,PUT mapping API>>.
 
@@ -24,10 +24,8 @@ PUT my_index/_mapping/my_type
 {
   "properties": {
     "title": {
-      "type": "string",
-      "norms": {
-        "enabled": false
-      }
+      "type": "text",
+      "norms": false
     }
   }
 }
@@ -41,31 +39,3 @@ results since some documents won't have norms anymore while other documents
 might still have norms.
 
 
-==== Lazy loading of norms
-
-Norms can be loaded into memory eagerly (`eager`), whenever a new segment
-comes online, or they can loaded lazily (`lazy`, default), only when the field
-is queried.
-
-Eager loading can be configured as follows:
-
-[source,js]
-------------
-PUT my_index/_mapping/my_type
-{
-  "properties": {
-    "title": {
-      "type": "string",
-      "norms": {
-        "loading": "eager"
-      }
-    }
-  }
-}
-------------
-// AUTOSENSE
-
-TIP: The `norms.loading` setting must have the same setting for fields of the
-same name in the same index.  Its value can be updated on existing fields
-using the <<indices-put-mapping,PUT mapping API>>.
-
diff --git a/docs/reference/mapping/params/null-value.asciidoc b/docs/reference/mapping/params/null-value.asciidoc
index 4d70d4a6ac5..b77a2e4da69 100644
--- a/docs/reference/mapping/params/null-value.asciidoc
+++ b/docs/reference/mapping/params/null-value.asciidoc
@@ -16,8 +16,7 @@ PUT my_index
     "my_type": {
       "properties": {
         "status_code": {
-          "type":       "string",
-          "index":      "not_analyzed",
+          "type":       "keyword",
           "null_value": "NULL" <1>
         }
       }
@@ -50,6 +49,4 @@ GET my_index/_search
 <3> A query for `NULL` returns document 1, but not document 2.
 
 IMPORTANT: The `null_value` needs to be the same datatype as the field.  For
-instance, a `long` field cannot have a string `null_value`.  String fields
-which are `analyzed` will also pass the `null_value` through the configured
-analyzer.
+instance, a `long` field cannot have a string `null_value`.
diff --git a/docs/reference/mapping/params/position-increment-gap.asciidoc b/docs/reference/mapping/params/position-increment-gap.asciidoc
index 962e2178469..d2cf1360080 100644
--- a/docs/reference/mapping/params/position-increment-gap.asciidoc
+++ b/docs/reference/mapping/params/position-increment-gap.asciidoc
@@ -57,7 +57,7 @@ PUT my_index
     "groups": {
       "properties": {
         "names": {
-          "type": "string",
+          "type": "text",
           "position_increment_gap": 0 <1>
         }
       }
diff --git a/docs/reference/mapping/params/properties.asciidoc b/docs/reference/mapping/params/properties.asciidoc
index 7683272ea19..a4f5277649a 100644
--- a/docs/reference/mapping/params/properties.asciidoc
+++ b/docs/reference/mapping/params/properties.asciidoc
@@ -23,14 +23,14 @@ PUT my_index
         "manager": { <2>
           "properties": {
             "age":  { "type": "integer" },
-            "name": { "type": "string"  }
+            "name": { "type": "text"  }
           }
         },
         "employees": { <3>
           "type": "nested",
           "properties": {
             "age":  { "type": "integer" },
-            "name": { "type": "string"  }
+            "name": { "type": "text"  }
           }
         }
       }
diff --git a/docs/reference/mapping/params/search-analyzer.asciidoc b/docs/reference/mapping/params/search-analyzer.asciidoc
index b1aa7b6c688..5a732ee3aac 100644
--- a/docs/reference/mapping/params/search-analyzer.asciidoc
+++ b/docs/reference/mapping/params/search-analyzer.asciidoc
@@ -41,7 +41,7 @@ PUT /my_index
     "my_type": {
       "properties": {
         "text": {
-          "type": "string",
+          "type": "text",
           "analyzer": "autocomplete", <2>
           "search_analyzer": "standard" <2>
         }
diff --git a/docs/reference/mapping/params/similarity.asciidoc b/docs/reference/mapping/params/similarity.asciidoc
index 0cdbd80a93e..731c557e730 100644
--- a/docs/reference/mapping/params/similarity.asciidoc
+++ b/docs/reference/mapping/params/similarity.asciidoc
@@ -5,8 +5,8 @@ Elasticsearch allows you to configure a scoring algorithm or _similarity_ per
 field. The `similarity` setting provides a simple way of choosing a similarity
 algorithm other than the default TF/IDF, such as `BM25`.
 
-Similarities are mostly useful for <<string,`string`>> fields, especially
-`analyzed` string fields, but can also apply to other field types.
+Similarities are mostly useful for <<text,`text`>> fields, but can also apply
+to other field types.
 
 Custom similarities can be configured by tuning the parameters of the built-in
 similarities. For more details about this expert options, see the
@@ -37,10 +37,10 @@ PUT my_index
     "my_type": {
       "properties": {
         "default_field": { <1>
-          "type": "string"
+          "type": "text"
         },
         "bm25_field": {
-          "type": "string",
+          "type": "text",
           "similarity": "BM25" <2>
         }
       }
diff --git a/docs/reference/mapping/params/store.asciidoc b/docs/reference/mapping/params/store.asciidoc
index b81208aed77..9f10b25724c 100644
--- a/docs/reference/mapping/params/store.asciidoc
+++ b/docs/reference/mapping/params/store.asciidoc
@@ -1,7 +1,7 @@
 [[mapping-store]]
 === `store`
 
-By default, field values <<mapping-index,indexed>> to make them searchable,
+By default, field values are <<mapping-index,indexed>> to make them searchable,
 but they are not _stored_.  This means that the field can be queried, but the
 original field value cannot be retrieved.
 
@@ -24,7 +24,7 @@ PUT /my_index
     "my_type": {
       "properties": {
         "title": {
-          "type": "string",
+          "type": "text",
           "store": true <1>
         },
         "date": {
@@ -32,7 +32,7 @@ PUT /my_index
           "store": true <1>
         },
         "content": {
-          "type": "string"
+          "type": "text"
         }
       }
     }
diff --git a/docs/reference/mapping/params/term-vector.asciidoc b/docs/reference/mapping/params/term-vector.asciidoc
index 74c4c416d95..136ab084149 100644
--- a/docs/reference/mapping/params/term-vector.asciidoc
+++ b/docs/reference/mapping/params/term-vector.asciidoc
@@ -35,7 +35,7 @@ PUT my_index
     "my_type": {
       "properties": {
         "text": {
-          "type":        "string",
+          "type":        "text",
           "term_vector": "with_positions_offsets"
         }
       }
diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc
index 2ac579f273a..30d6bd56b1f 100644
--- a/docs/reference/mapping/types.asciidoc
+++ b/docs/reference/mapping/types.asciidoc
@@ -7,7 +7,7 @@ document:
 [float]
 === Core datatypes
 
-<<string>>::    `string`
+string::        <<text,`text`>> and <<keyword,`keyword`>>
 <<number>>::    `long`, `integer`, `short`, `byte`, `double`, `float`
 <<date>>::      `date`
 <<boolean>>::   `boolean`
@@ -45,9 +45,9 @@ Attachment datatype::
 === Multi-fields
 
 It is often useful to index the same field in different ways for different
-purposes. For instance, a `string` field could be <<mapping-index,indexed>> as
-an `analyzed` field for full-text search, and as a `not_analyzed` field for
-sorting or aggregations.  Alternatively, you could index a string field with
+purposes. For instance, a `string` field could be mapped as
+a `text` field for full-text search, and as a `keyword` field for
+sorting or aggregations.  Alternatively, you could index a text field with
 the <<analysis-standard-analyzer,`standard` analyzer>>, the
 <<english-analyzer,`english`>> analyzer, and the
 <<french-analyzer,`french` analyzer>>.
@@ -69,6 +69,8 @@ include::types/geo-shape.asciidoc[]
 
 include::types/ip.asciidoc[]
 
+include::types/keyword.asciidoc[]
+
 include::types/nested.asciidoc[]
 
 include::types/numeric.asciidoc[]
@@ -77,6 +79,8 @@ include::types/object.asciidoc[]
 
 include::types/string.asciidoc[]
 
+include::types/text.asciidoc[]
+
 include::types/token-count.asciidoc[]
 
 
diff --git a/docs/reference/mapping/types/binary.asciidoc b/docs/reference/mapping/types/binary.asciidoc
index 4e5f6b4bc27..7f82523416f 100644
--- a/docs/reference/mapping/types/binary.asciidoc
+++ b/docs/reference/mapping/types/binary.asciidoc
@@ -13,7 +13,7 @@ PUT my_index
     "my_type": {
       "properties": {
         "name": {
-          "type": "string"
+          "type": "text"
         },
         "blob": {
           "type": "binary"
diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc
index e724a5428b2..45c5e65addb 100644
--- a/docs/reference/mapping/types/geo-point.asciidoc
+++ b/docs/reference/mapping/types/geo-point.asciidoc
@@ -101,17 +101,6 @@ The following parameters are accepted by `geo_point` fields:
 
 [horizontal]
 
-<<coerce,`coerce`>>::
-
-    Normalize longitude and latitude values to a standard -180:180 / -90:90
-    coordinate system. Accepts `true` and `false` (default).
-
-<<doc-values,`doc_values`>>::
-
-    Should the field be stored on disk in a column-stride fashion, so that it
-    can later be used for sorting, aggregations, or scripting? Accepts `true`
-    (default) or `false`.
-
 <<geohash,`geohash`>>::
 
     Should the geo-point also be indexed as a geohash in the `.geohash`
diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc
new file mode 100644
index 00000000000..66c7135c37d
--- /dev/null
+++ b/docs/reference/mapping/types/keyword.asciidoc
@@ -0,0 +1,111 @@
+[[keyword]]
+=== Keyword datatype
+
+A field to index structured content such as email addresses, hostnames, status
+codes, zip codes or tags.
+
+They are typically used for filtering (_Find me all blog posts where
+++status++ is ++published++_), for sorting, and for aggregations. Keyword
+fields are ony searchable by their exact value.
+
+If you need to index full text content such as email bodies or product
+descriptions, it is likely that you should rather use a <<text,`text`>> field.
+
+Below is an example of a mapping for a keyword field:
+
+[source,js]
+--------------------------------
+PUT my_index
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "tags": {
+          "type":  "keyword"
+        }
+      }
+    }
+  }
+}
+--------------------------------
+// AUTOSENSE
+
+[[keyword-params]]
+==== Parameters for keyword fields
+
+The following parameters are accepted by `string` fields:
+
+[horizontal]
+
+<<mapping-boost,`boost`>>::
+
+    Mapping field-level query time boosting. Accepts a floating point number, defaults
+    to `1.0`.
+
+<<doc-values,`doc_values`>>::
+
+    Should the field be stored on disk in a column-stride fashion, so that it
+    can later be used for sorting, aggregations, or scripting? Accepts `true`
+    (default) or `false`.
+
+<<global-ordinals,`eager_global_ordinals`>>::
+
+    Should global ordinals be loaded eagerly on refresh? Accepts `true` or `false`
+    (default). Enabling this is a good idea on fields that are frequently used for
+    terms aggregations.
+
+<<multi-fields,`fields`>>::
+
+    Multi-fields allow the same string value to be indexed in multiple ways for
+    different purposes, such as one field for search and a multi-field for
+    sorting and aggregations.
+
+<<ignore-above,`ignore_above`>>::
+
+    Do not index or analyze any string longer than this value.  Defaults to
+    `2147483647` so that all values would be accepted.
+
+<<include-in-all,`include_in_all`>>::
+
+    Whether or not the field value should be included in the
+    <<mapping-all-field,`_all`>> field? Accepts `true` or `false`.  Defaults
+    to `false` if <<mapping-index,`index`>> is set to `no`, or if a parent
+    <<object,`object`>> field sets `include_in_all` to `false`.
+    Otherwise defaults to `true`.
+
+<<mapping-index,`index`>>::
+
+    Should the field be searchable? Accepts `true` (default) or `false`.
+
+<<index-options,`index_options`>>::
+
+    What information should be stored in the index, for scoring purposes.
+    Defaults to `docs` but can also be set to `freqs` to take term frequency into account
+    when computing scores.
+
+<<norms,`norms`>>::
+
+    Whether field-length should be taken into account when scoring queries.
+    Accepts `true` or `false` (default).
+
+<<null-value,`null_value`>>::
+
+    Accepts a string value which is substituted for any explicit `null`
+    values.  Defaults to `null`, which means the field is treated as missing.
+
+<<mapping-store,`store`>>::
+
+    Whether the field value should be stored and retrievable separately from
+    the <<mapping-source-field,`_source`>> field. Accepts `true` or `false`
+    (default).
+
+<<search-analyzer,`search_analyzer`>>::
+
+    The <<analyzer,`analyzer`>> that should be used at search time on
+    <<mapping-index,`analyzed`>> fields. Defaults to the `analyzer` setting.
+
+<<similarity,`similarity`>>::
+
+    Which scoring algorithm or _similarity_ should be used. Defaults
+    to `classic`, which uses TF/IDF.
+
diff --git a/docs/reference/mapping/types/object.asciidoc b/docs/reference/mapping/types/object.asciidoc
index 0d159d7e1ef..6d35e6aee97 100644
--- a/docs/reference/mapping/types/object.asciidoc
+++ b/docs/reference/mapping/types/object.asciidoc
@@ -46,16 +46,15 @@ PUT my_index
     "my_type": { <1>
       "properties": {
         "region": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword"
         },
         "manager": { <2>
           "properties": {
             "age":  { "type": "integer" },
             "name": { <3>
               "properties": {
-                "first": { "type": "string" },
-                "last":  { "type": "string" }
+                "first": { "type": "text" },
+                "last":  { "type": "text" }
               }
             }
           }
diff --git a/docs/reference/mapping/types/string.asciidoc b/docs/reference/mapping/types/string.asciidoc
index 6ff78aa4732..88ca4e1d920 100644
--- a/docs/reference/mapping/types/string.asciidoc
+++ b/docs/reference/mapping/types/string.asciidoc
@@ -1,179 +1,4 @@
 [[string]]
 === String datatype
 
-Fields of type `string` accept text values.  Strings may be sub-divided into:
-
-Full text::
-+
---
-
-Full text values, like the body of an email, are typically used for text based
-relevance searches, such as: _Find the most relevant documents that match a
-query for "quick brown fox"_.
-
-These fields are `analyzed`, that is they are passed through an
-<<analysis,analyzer>> to convert the string into a list of individual terms
-before being indexed. The analysis process allows Elasticsearch to search for
-individual words _within_  each full text field.  Full text fields are not
-used for sorting and seldom used for aggregations (although the
-<<search-aggregations-bucket-significantterms-aggregation,significant terms aggregation>> is a notable exception).
-
---
-
-Keywords::
-
-Keywords are exact values like email addresses, hostnames, status codes, or
-tags.  They are typically used for filtering (_Find me all blog posts where
-++status++ is ++published++_), for sorting, and for aggregations. Keyword
-fields are `not_analyzed`.  Instead, the exact string value is added to the
-index as a single term.
-
-Below is an example of a mapping for a full text (`analyzed`) and a keyword
-(`not_analyzed`) string field:
-
-[source,js]
---------------------------------
-PUT my_index
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "full_name": { <1>
-          "type":  "string"
-        },
-        "status": {
-          "type":  "string", <2>
-          "index": "not_analyzed"
-        }
-      }
-    }
-  }
-}
---------------------------------
-// AUTOSENSE
-<1> The `full_name` field is an `analyzed` full text field -- `index:analyzed` is the default.
-<2> The `status` field is a `not_analyzed` keyword field.
-
-Sometimes it is useful to have both a full text (`analyzed`) and a keyword
-(`not_analyzed`) version of the same field: one for full text search and the
-other for aggregations and sorting. This can be achieved with
-<<multi-fields,multi-fields>>.
-
-
-[[string-params]]
-==== Parameters for string fields
-
-The following parameters are accepted by `string` fields:
-
-[horizontal]
-
-<<analyzer,`analyzer`>>::
-
-    The <<analysis,analyzer>> which should be used for
-    <<mapping-index,`analyzed`>> string fields, both at index-time and at
-    search-time (unless overridden by the  <<search-analyzer,`search_analyzer`>>).
-    Defaults to the default index analyzer, or the
-    <<analysis-standard-analyzer,`standard` analyzer>>.
-
-<<mapping-boost,`boost`>>::
-
-    Mapping field-level query time boosting. Accepts a floating point number, defaults
-    to `1.0`.
-
-<<doc-values,`doc_values`>>::
-
-    Should the field be stored on disk in a column-stride fashion, so that it
-    can later be used for sorting, aggregations, or scripting? Accepts `true`
-    or `false`. Defaults to `true` for `not_analyzed` fields. Analyzed fields
-    do not support doc values.
-
-<<fielddata,`fielddata`>>::
-
-    Can the field use in-memory fielddata for sorting, aggregations,
-    or scripting? Accepts `disabled` or `paged_bytes` (default).
-    Not analyzed fields will use <<doc-values,doc values>> in preference
-    to fielddata.
-
-<<multi-fields,`fields`>>::
-
-    Multi-fields allow the same string value to be indexed in multiple ways for
-    different purposes, such as one field for search and a multi-field for
-    sorting and aggregations, or the same string value analyzed by different
-    analyzers.
-
-<<ignore-above,`ignore_above`>>::
-
-    Do not index or analyze any string longer than this value.  Defaults to `0` (disabled).
-
-<<include-in-all,`include_in_all`>>::
-
-    Whether or not the field value should be included in the
-    <<mapping-all-field,`_all`>> field? Accepts `true` or `false`.  Defaults
-    to `false` if <<mapping-index,`index`>> is set to `no`, or if a parent
-    <<object,`object`>> field sets `include_in_all` to `false`.
-    Otherwise defaults to `true`.
-
-<<mapping-index,`index`>>::
-
-    Should the field be searchable? Accepts `analyzed` (default, treat as full-text field),
-    `not_analyzed` (treat as keyword field) and `no`.
-
-<<index-options,`index_options`>>::
-
-    What information should be stored in the index, for search and highlighting purposes.
-    Defaults to `positions` for <<mapping-index,`analyzed`>> fields, and to `docs` for
-    `not_analyzed` fields.
-
-
-<<norms,`norms`>>::
-+
---
-
-Whether field-length should be taken into account when scoring queries.
-Defaults depend on the <<mapping-index,`index`>> setting:
-
-* `analyzed` fields default to `{ "enabled": true, "loading": "lazy" }`.
-* `not_analyzed` fields default to `{ "enabled": false }`.
---
-
-<<null-value,`null_value`>>::
-
-    Accepts a string value which is substituted for any explicit `null`
-    values.  Defaults to `null`, which means the field is treated as missing.
-    If the field is `analyzed`, the `null_value` will also be analyzed.
-
-<<position-increment-gap,`position_increment_gap`>>::
-
-    The number of fake term positions which should be inserted between
-    each element of an array of strings. Defaults to 0.
-    The number of fake term position which should be inserted between each
-    element of an array of strings. Defaults to the position_increment_gap
-    configured on the analyzer which defaults to 100. 100 was chosen because it
-    prevents phrase queries with reasonably large slops (less than 100) from
-    matching terms across field values.
-
-<<mapping-store,`store`>>::
-
-    Whether the field value should be stored and retrievable separately from
-    the <<mapping-source-field,`_source`>> field. Accepts `true` or `false`
-    (default).
-
-<<search-analyzer,`search_analyzer`>>::
-
-    The <<analyzer,`analyzer`>> that should be used at search time on
-    <<mapping-index,`analyzed`>> fields. Defaults to the `analyzer` setting.
-	
-<<search-quote-analyzer,`search_quote_analyzer`>>::
-
-    The <<analyzer,`analyzer`>> that should be used at search time when a
-    phrase is encountered. Defaults to the `search_analyzer` setting.
-
-<<similarity,`similarity`>>::
-
-    Which scoring algorithm or _similarity_ should be used. Defaults
-    to `classic`, which uses TF/IDF.
-
-<<term-vector,`term_vector`>>::
-
-    Whether term vectors should be stored for an <<mapping-index,`analyzed`>>
-    field. Defaults to `no`.
+NOTE: The `string` field has been removed in favor of the `text` and `keyword` fields.
diff --git a/docs/reference/mapping/types/text.asciidoc b/docs/reference/mapping/types/text.asciidoc
new file mode 100644
index 00000000000..7798b2c41f0
--- /dev/null
+++ b/docs/reference/mapping/types/text.asciidoc
@@ -0,0 +1,139 @@
+[[text]]
+=== Text datatype
+
+A field to index full-text values, such as the body of on email or the
+description of a product. These fields are `analyzed`, that is they are passed through an
+<<analysis,analyzer>> to convert the string into a list of individual terms
+before being indexed. The analysis process allows Elasticsearch to search for
+individual words _within_  each full text field.  Text fields are not
+used for sorting and seldom used for aggregations (although the
+<<search-aggregations-bucket-significantterms-aggregation,significant terms aggregation>> 
+is a notable exception).
+
+If you need to index structured content such as email addresses, hostnames, status
+codes, or tags, it is likely that you should rather use a <<keyword,`keyword`>> field.
+
+Below is an example of a mapping for a text field:
+
+[source,js]
+--------------------------------
+PUT my_index
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "full_name": {
+          "type":  "text"
+        }
+      }
+    }
+  }
+}
+--------------------------------
+// AUTOSENSE
+
+Sometimes it is useful to have both a full text (`text`) and a keyword
+(`keyword`) version of the same field: one for full text search and the
+other for aggregations and sorting. This can be achieved with
+<<multi-fields,multi-fields>>.
+
+[[text-params]]
+==== Parameters for text fields
+
+The following parameters are accepted by `text` fields:
+
+[horizontal]
+
+<<analyzer,`analyzer`>>::
+
+    The <<analysis,analyzer>> which should be used for
+    <<mapping-index,`analyzed`>> string fields, both at index-time and at
+    search-time (unless overridden by the  <<search-analyzer,`search_analyzer`>>).
+    Defaults to the default index analyzer, or the
+    <<analysis-standard-analyzer,`standard` analyzer>>.
+
+<<mapping-boost,`boost`>>::
+
+    Mapping field-level query time boosting. Accepts a floating point number, defaults
+    to `1.0`.
+
+<<global-ordinals,`eager_global_ordinals`>>::
+
+    Should global ordinals be loaded eagerly on refresh? Accepts `true` or `false`
+    (default). Enabling this is a good idea on fields that are frequently used for
+    (significant) terms aggregations.
+
+<<fielddata,`fielddata`>>::
+
+    Can the field use in-memory fielddata for sorting, aggregations,
+    or scripting? Accepts `true` or `false` (default).
+
+<<field-data-filtering,`fielddata_frequency_filter`>>::
+
+    Expert settings which allow to decide which values to load in memory when `fielddata`
+    is enabled. By default all values are loaded.
+
+<<multi-fields,`fields`>>::
+
+    Multi-fields allow the same string value to be indexed in multiple ways for
+    different purposes, such as one field for search and a multi-field for
+    sorting and aggregations, or the same string value analyzed by different
+    analyzers.
+
+<<include-in-all,`include_in_all`>>::
+
+    Whether or not the field value should be included in the
+    <<mapping-all-field,`_all`>> field? Accepts `true` or `false`.  Defaults
+    to `false` if <<mapping-index,`index`>> is set to `no`, or if a parent
+    <<object,`object`>> field sets `include_in_all` to `false`.
+    Otherwise defaults to `true`.
+
+<<mapping-index,`index`>>::
+
+    Should the field be searchable? Accepts `true` (default) or `false`.
+
+<<index-options,`index_options`>>::
+
+    What information should be stored in the index, for search and highlighting purposes.
+    Defaults to `positions`.
+
+<<norms,`norms`>>::
+
+    Whether field-length should be taken into account when scoring queries.
+    Accepts `true` (default) or `false`.
+
+<<position-increment-gap,`position_increment_gap`>>::
+
+    The number of fake term positions which should be inserted between
+    each element of an array of strings. Defaults to 0.
+    The number of fake term position which should be inserted between each
+    element of an array of strings. Defaults to the position_increment_gap
+    configured on the analyzer which defaults to 100. 100 was chosen because it
+    prevents phrase queries with reasonably large slops (less than 100) from
+    matching terms across field values.
+
+<<mapping-store,`store`>>::
+
+    Whether the field value should be stored and retrievable separately from
+    the <<mapping-source-field,`_source`>> field. Accepts `true` or `false`
+    (default).
+
+<<search-analyzer,`search_analyzer`>>::
+
+    The <<analyzer,`analyzer`>> that should be used at search time on
+    <<mapping-index,`analyzed`>> fields. Defaults to the `analyzer` setting.
+
+<<search-quote-analyzer,`search_quote_analyzer`>>::
+
+    The <<analyzer,`analyzer`>> that should be used at search time when a
+    phrase is encountered. Defaults to the `search_analyzer` setting.
+
+<<similarity,`similarity`>>::
+
+    Which scoring algorithm or _similarity_ should be used. Defaults
+    to `classic`, which uses TF/IDF.
+
+<<term-vector,`term_vector`>>::
+
+    Whether term vectors should be stored for an <<mapping-index,`analyzed`>>
+    field. Defaults to `no`.
diff --git a/docs/reference/mapping/types/token-count.asciidoc b/docs/reference/mapping/types/token-count.asciidoc
index ca2ed6f0428..baa6d409ead 100644
--- a/docs/reference/mapping/types/token-count.asciidoc
+++ b/docs/reference/mapping/types/token-count.asciidoc
@@ -15,7 +15,7 @@ PUT my_index
     "my_type": {
       "properties": {
         "name": { <1>
-          "type": "string",
+          "type": "text",
           "fields": {
             "length": { <2>
               "type":     "token_count",
diff --git a/docs/reference/migration/index.asciidoc b/docs/reference/migration/index.asciidoc
index 57401cb01d7..0dda43da713 100644
--- a/docs/reference/migration/index.asciidoc
+++ b/docs/reference/migration/index.asciidoc
@@ -8,26 +8,15 @@ your application from one version of Elasticsearch to another.
 
 As a general rule:
 
-* Migration between major versions -- e.g. `1.x` to `2.x` --
+* Migration between minor versions -- e.g. `5.x` to `5.y` -- can be
+  performed by <<rolling-upgrades,upgrading one node at a time>>.
+
+* Migration between consecutive major versions -- e.g. `2.x` to `5.x` --
   requires a <<restart-upgrade,full cluster restart>>.
 
-* Migration between minor versions -- e.g. `1.x` to `1.y` -- can be
-  performed by <<rolling-upgrades,upgrading one node at a time>>.
+* Migration between non-consecutive major versions -- e.g. `1.x` to `5.x` --
+  is not supported.
 
 See <<setup-upgrade>> for more info.
 --
 include::migrate_5_0.asciidoc[]
-
-include::migrate_2_3.asciidoc[]
-
-include::migrate_2_2.asciidoc[]
-
-include::migrate_2_1.asciidoc[]
-
-include::migrate_2_0.asciidoc[]
-
-include::migrate_1_6.asciidoc[]
-
-include::migrate_1_4.asciidoc[]
-
-include::migrate_1_0.asciidoc[]
diff --git a/docs/reference/migration/migrate_1_0.asciidoc b/docs/reference/migration/migrate_1_0.asciidoc
deleted file mode 100644
index 1e917c4a0d9..00000000000
--- a/docs/reference/migration/migrate_1_0.asciidoc
+++ /dev/null
@@ -1,372 +0,0 @@
-[[breaking-changes-1.0]]
-== Breaking changes in 1.0
-
-This section discusses the changes that you need to be aware of when migrating
-your application to Elasticsearch 1.0.
-
-=== System and settings
-
-* Elasticsearch now runs in the foreground by default.  There is no more `-f`
-  flag on the command line.  Instead, to run elasticsearch as a daemon, use
-  the `-d` flag:
-
-[source,sh]
----------------
-./bin/elasticsearch -d
----------------
-
-* Command line settings can now be passed without the `-Des.` prefix, for
-  instance:
-
-[source,sh]
----------------
-./bin/elasticsearch --node.name=search_1 --cluster.name=production
----------------
-
-* Elasticsearch on 64 bit Linux now uses <<mmapfs,`mmapfs`>> by default.  Make
-  sure that you set <<setup-service,`MAX_MAP_COUNT`>> to a sufficiently high
-  number.  The RPM and Debian packages default this value to `262144`.
-
-* The RPM and Debian packages no longer start Elasticsearch by default.
-
-* The `cluster.routing.allocation` settings (`disable_allocation`,
-  `disable_new_allocation` and `disable_replica_location`) have been
-  <<modules-cluster,replaced by the single setting>>:
-+
-[source,yaml]
----------------
-cluster.routing.allocation.enable: all|primaries|new_primaries|none
----------------
-
-=== Stats and Info APIs
-
-The <<cluster-state,`cluster_state`>>, <<cluster-nodes-info,`nodes_info`>>,
-<<cluster-nodes-stats,`nodes_stats`>> and <<indices-stats,`indices_stats`>>
-APIs have all been changed to make their format more RESTful and less clumsy.
-
-For instance, if you just want the `nodes` section of the `cluster_state`,
-instead of:
-
-[source,sh]
----------------
-GET /_cluster/state?filter_metadata&filter_routing_table&filter_blocks
----------------
-
-you now use:
-
-[source,sh]
----------------
-GET /_cluster/state/nodes
----------------
-
-Similarly for the `nodes_stats` API, if you want the `transport` and `http`
-metrics only, instead of:
-
-[source,sh]
----------------
-GET /_nodes/stats?clear&transport&http
----------------
-
-you now use:
-
-[source,sh]
----------------
-GET /_nodes/stats/transport,http
----------------
-
-See the links above for full details.
-
-
-=== Indices APIs
-
-The `mapping`, `alias`, `settings`, and `warmer` index APIs are all similar
-but there are subtle differences in the order of the URL and the response
-body.  For instance, adding a mapping and a warmer look slightly different:
-
-[source,sh]
----------------
-PUT /{index}/{type}/_mapping
-PUT /{index}/_warmer/{name}
----------------
-
-These URLs have been unified as:
-
-[source,sh]
----------------
-PUT /{indices}/_mapping/{type}
-PUT /{indices}/_alias/{name}
-PUT /{indices}/_warmer/{name}
-
-GET /{indices}/_mapping/{types}
-GET /{indices}/_alias/{names}
-GET /{indices}/_settings/{names}
-GET /{indices}/_warmer/{names}
-
-DELETE /{indices}/_mapping/{types}
-DELETE /{indices}/_alias/{names}
-DELETE /{indices}/_warmer/{names}
----------------
-
-All of the `{indices}`, `{types}` and `{names}` parameters can be replaced by:
-
-  * `_all`, `*` or blank (ie left out altogether), all of which mean ``all''
-  * wildcards like `test*`
-  * comma-separated lists: `index_1,test_*`
-
-The only exception is `DELETE` which doesn't accept blank (missing)
-parameters. If you want to delete something, you should be specific.
-
-Similarly, the return values for `GET` have been unified with the following
-rules:
-
-* Only return values that exist.  If you try to `GET` a mapping which doesn't
-  exist, then the result will be an empty object: `{}`. We no longer throw a
-  `404` if the requested mapping/warmer/alias/setting doesn't exist.
-
-* The response format always has the index name, then the section, then the
-  element name, for instance:
-+
-[source,js]
----------------
-{
-    "my_index": {
-        "mappings": {
-            "my_type": {...}
-        }
-    }
-}
----------------
-+
-This is a breaking change for the `get_mapping` API.
-
-In the future we will also provide plural versions to allow putting multiple mappings etc in a single request.
-
-See <<indices-put-mapping,`put-mapping`>>, <<indices-get-mapping,`get-
-mapping`>>, <<indices-get-field-mapping,`get-field-mapping`>>,
-<<indices-update-settings,`update-settings`>>, <<indices-get-settings,`get-settings`>>,
-`warmers`, and <<indices-aliases,`aliases`>> for more details.
-
-=== Index request
-
-Previously a document could be indexed as itself, or wrapped in an outer
-object which specified the `type` name:
-
-[source,js]
----------------
-PUT /my_index/my_type/1
-{
-  "my_type": {
-     ... doc fields ...
-  }
-}
----------------
-
-This led to some ambiguity when a document also included a field with the same
-name as the `type`.  We no longer accept the outer `type` wrapper, but this
-behaviour can be reenabled on an index-by-index basis with the setting:
-`index.mapping.allow_type_wrapper`.
-
-=== Search requests
-
-While the `search` API takes a top-level `query` parameter, the
-<<search-count,`count`>>, `delete-by-query` and
-<<search-validate,`validate-query`>> requests expected the whole body to be a
-query. These now _require_ a top-level `query` parameter:
-
-[source,js]
----------------
-GET /_count
-{
-    "query": {
-        "match": {
-            "title": "Interesting stuff"
-        }
-    }
-}
----------------
-
-Also, the top-level `filter` parameter in search has been renamed to
-<<search-request-post-filter,`post_filter`>>, to indicate that it should not
-be used as the primary way to filter search results (use a
-<<query-dsl-bool-query,`bool` query>> instead), but only to filter
-results AFTER aggregations have been calculated.
-
-This example counts the top colors in all matching docs, but only returns docs
-with color `red`:
-
-[source,js]
----------------
-GET /_search
-{
-    "query": {
-        "match_all": {}
-    },
-    "aggs": {
-        "colors": {
-            "terms": { "field": "color" }
-        }
-    },
-    "post_filter": {
-        "term": {
-            "color": "red"
-        }
-    }
-}
----------------
-
-=== Multi-fields
-
-Multi-fields are dead! Long live multi-fields!  Well, the field type
-`multi_field` has been removed.  Instead, any of the core field types
-(excluding `object` and `nested`) now accept a `fields` parameter.  It's the
-same thing, but nicer. Instead of:
-
-[source,js]
----------------
-"title": {
-    "type": "multi_field",
-    "fields": {
-        "title": { "type": "string" },
-        "raw":   { "type": "string", "index": "not_analyzed" }
-    }
-}
----------------
-
-you can now write:
-
-[source,js]
----------------
-"title": {
-    "type": "string",
-    "fields": {
-        "raw":   { "type": "string", "index": "not_analyzed" }
-    }
-}
----------------
-
-Existing multi-fields will be upgraded to the new format automatically.
-
-Also, instead of having to use the arcane `path` and `index_name` parameters
-in order to index multiple fields into a single ``custom +_all+ field'', you
-can now use the <<copy-to,`copy_to` parameter>>.
-
-=== Stopwords
-
-Previously, the <<analysis-standard-analyzer,`standard`>> and
-<<analysis-pattern-analyzer,`pattern`>> analyzers used the list of English stopwords
-by default, which caused some hard to debug indexing issues.  Now they are set to
-use the empty stopwords list (ie `_none_`) instead.
-
-=== Dates without years
-
-When dates are specified without a year, for example: `Dec 15 10:00:00` they
-are treated as dates in 2000 during indexing and range searches... except for
-the upper included bound `lte` where they were treated as dates in 1970!  Now,
-all https://github.com/elastic/elasticsearch/issues/4451[dates without years]
-use `1970` as the default.
-
-=== Parameters
-
-* Geo queries used to use `miles` as the default unit.  And we
-  http://en.wikipedia.org/wiki/Mars_Climate_Orbiter[all know what
-  happened at NASA] because of that decision.  The new default unit is
-  https://github.com/elastic/elasticsearch/issues/4515[`meters`].
-
-* For all queries that support _fuzziness_, the `min_similarity`, `fuzziness`
-  and `edit_distance` parameters have been unified as the single parameter
-  `fuzziness`.  See <<fuzziness>> for details of accepted values.
-
-* The `ignore_missing` parameter has been replaced by the `expand_wildcards`,
-  `ignore_unavailable` and `allow_no_indices` parameters, all of which have
-  sensible defaults.  See <<multi-index,the multi-index docs>> for more.
-
-* An index name (or pattern) is now required for destructive operations like
-  deleting indices:
-+
-[source,sh]
----------------
-# v0.90 - delete all indices:
-DELETE /
-
-# v1.0 - delete all indices:
-DELETE /_all
-DELETE /*
----------------
-+
-Setting `action.destructive_requires_name` to `true` provides further safety
-by disabling wildcard expansion on destructive actions.
-
-=== Return values
-
-* The `ok` return value has been removed from all response bodies as it added
-  no useful information.
-
-* The `found`, `not_found` and `exists` return values have been unified as
-  `found` on all relevant APIs.
-
-* Field values, in response to the <<search-request-fields,`fields`>>
-  parameter, are now always returned as arrays.  A field could have single or
-  multiple values, which meant that sometimes they were returned as scalars
-  and sometimes as arrays.  By always returning arrays, this simplifies user
-  code.  The only exception to this rule is when `fields` is used to retrieve
-  metadata like the `routing` value, which are always singular.  Metadata
-  fields are always returned as scalars.
-+
-The `fields` parameter is intended to be used for retrieving stored fields,
-rather than for fields extracted from the `_source`. That means that it can no
-longer be used to return whole objects and it no longer accepts the
-`_source.fieldname` format. For these you should use the
-<<search-request-source-filtering,`_source`&#32; `_source_include` and `_source_exclude`>>
-parameters instead.
-
-* Settings, like `index.analysis.analyzer.default` are now returned as proper
-  nested JSON objects, which makes them easier to work with programmatically:
-+
-[source,js]
----------------
-{
-    "index": {
-        "analysis": {
-            "analyzer": {
-                "default": xxx
-            }
-        }
-    }
-}
----------------
-+
-You can choose to return them in flattened format by passing `?flat_settings`
-in the query string.
-
-* The <<indices-analyze,`analyze`>> API no longer supports the text response
-  format, but does support JSON and YAML.
-
-=== Deprecations
-
-* The `text` query has been removed.  Use the
-  <<query-dsl-match-query,`match`>> query instead.
-
-* The `field` query has been removed.  Use the
-  <<query-dsl-query-string-query,`query_string`>> query instead.
-
-* Per-document boosting with the `_boost` field has
-  been removed.  You can use the
-  <<query-dsl-function-score-query,`function_score`>> instead.
-
-* The `path` parameter in mappings has been deprecated. Use the
-  <<copy-to,`copy_to`>> parameter instead.
-
-* The `custom_score` and `custom_boost_score` is no longer supported. You can
-  use <<query-dsl-function-score-query,`function_score`>> instead.
-
-=== Percolator
-
-The percolator has been redesigned and because of this the dedicated `_percolator` index is no longer used by the percolator,
-but instead the percolator works with a dedicated `.percolator` type. Read the http://www.elastic.co/blog/percolator-redesign-blog-post[redesigned percolator]
-blog post for the reasons why the percolator has been redesigned.
-
-Elasticsearch will *not* delete the `_percolator` index when upgrading, only the percolate api will not use the queries
-stored in the `_percolator` index. In order to use the already stored queries, you can just re-index the queries from the
-`_percolator` index into any index under the reserved `.percolator` type. The format in which the percolate queries
-were stored has *not* been changed. So a simple script that does a scan search to retrieve all the percolator queries
-and then does a bulk request into another index should be sufficient.
diff --git a/docs/reference/migration/migrate_1_4.asciidoc b/docs/reference/migration/migrate_1_4.asciidoc
deleted file mode 100644
index c20504bbddf..00000000000
--- a/docs/reference/migration/migrate_1_4.asciidoc
+++ /dev/null
@@ -1,92 +0,0 @@
-[[breaking-changes-1.4]]
-== Breaking changes in 1.4
-
-This section discusses the changes that you need to be aware of when migrating
-your application from Elasticsearch 1.x to Elasticsearch 1.4.
-
-[float]
-=== Percolator
-
-In indices created with version `1.4.0` or later, percolation queries can only
-refer to fields that already exist in the mappings in that index. There are
-two ways to make sure that a field mapping exist:
-
-* Add or update a mapping via the <<indices-create-index,create index>> or
-  <<indices-put-mapping,put mapping>> apis.
-* Percolate a document before registering a query. Percolating a document can
-  add field mappings dynamically, in the same way as happens when indexing a
-  document.
-
-[float]
-=== Aliases
-
-<<indices-aliases,Aliases>> can include <<query-dsl,filters>> which
-are automatically applied to any search performed via the alias.
-<<filtered,Filtered aliases>> created with version `1.4.0` or later can only
-refer to field names which exist in the mappings of the index (or indices)
-pointed to by the alias.
-
-Add or update a mapping via the <<indices-create-index,create index>> or
-<<indices-put-mapping,put mapping>> apis.
-
-[float]
-=== Indices APIs
-
-The get warmer api will return a section for `warmers` even if there are
-no warmers.  This ensures that the following two examples are equivalent:
-
-[source,js]
---------------------------------------------------
-curl -XGET 'http://localhost:9200/_all/_warmers'
-
-curl -XGET 'http://localhost:9200/_warmers'
---------------------------------------------------
-
-The <<alias-retrieving, get alias api>> will return a section for `aliases` even if there are
-no aliases.  This ensures that the following two examples are equivalent:
-
-[source,js]
---------------------------------------------------
-curl -XGET 'http://localhost:9200/_all/_aliases'
-
-curl -XGET 'http://localhost:9200/_aliases'
---------------------------------------------------
-
-The <<indices-get-mapping, get mapping api>> will return a section for `mappings` even if there are
-no mappings.  This ensures that the following two examples are equivalent:
-
-[source,js]
---------------------------------------------------
-curl -XGET 'http://localhost:9200/_all/_mappings'
-
-curl -XGET 'http://localhost:9200/_mappings'
---------------------------------------------------
-
-[float]
-=== Bulk UDP
-
-Bulk UDP has been deprecated and will be removed in 2.0.
-You should use <<docs-bulk,the standard bulk API>> instead.
-Each cluster must have an elected master node in order to be fully operational. Once a node loses its elected master
-node it will reject some or all operations.
-
-[float]
-=== Zen discovery
-
-On versions before `1.4.0.Beta1` all operations are rejected when a node loses its elected master. From `1.4.0.Beta1`
-only write operations will be rejected by default. Read operations will still be served based on the information available
-to the node, which may result in being partial and possibly also stale. If the default is undesired then the
-pre `1.4.0.Beta1` behaviour can be enabled, see: <<modules-discovery-zen,no-master-block>>
-
-[float]
-=== More Like This Field
-
-The More Like This Field query has been deprecated in favor of the <<query-dsl-mlt-query, More Like This Query>>
-restrained set to a specific `field`. It will be removed in 2.0.
-
-[float]
-=== MVEL is deprecated
-
-Groovy is the new default scripting language in Elasticsearch, and is enabled in `sandbox` mode
-by default.  MVEL has been removed from core, but is available as a plugin:
-https://github.com/elastic/elasticsearch-lang-mvel
diff --git a/docs/reference/migration/migrate_1_6.asciidoc b/docs/reference/migration/migrate_1_6.asciidoc
deleted file mode 100644
index 9540d3b6759..00000000000
--- a/docs/reference/migration/migrate_1_6.asciidoc
+++ /dev/null
@@ -1,17 +0,0 @@
-[[breaking-changes-1.6]]
-== Breaking changes in 1.6
-
-This section discusses the changes that you need to be aware of when migrating
-your application from Elasticsearch 1.x to Elasticsearch 1.6.
-
-[float]
-=== More Like This API
-
-The More Like This API query has been deprecated and will be removed in 2.0. Instead use the <<query-dsl-mlt-query, More Like This Query>>.
-
-[float]
-=== `top_children` query
-
-The `top_children` query has been deprecated and will be removed in 2.0. Instead the `has_child` query should be used.
-The `top_children` query isn't always faster than the `has_child` query and the `top_children` query is often inaccurate.
-The total hits and any aggregations in the same search request will likely be off.
diff --git a/docs/reference/migration/migrate_2_0.asciidoc b/docs/reference/migration/migrate_2_0.asciidoc
deleted file mode 100644
index adf12e7da5c..00000000000
--- a/docs/reference/migration/migrate_2_0.asciidoc
+++ /dev/null
@@ -1,73 +0,0 @@
-[[breaking-changes-2.0]]
-== Breaking changes in 2.0
-
-This section discusses the changes that you need to be aware of when migrating
-your application to Elasticsearch 2.0.
-
-[float]
-=== Indices created before 0.90
-
-Elasticsearch 2.0 can read indices created in version 0.90 and above.  If any
-of your indices were created before 0.90 you will need to upgrade to the
-latest 1.x version of Elasticsearch first, in order to upgrade your indices or
-to delete the old indices. Elasticsearch will not start in the presence of old
-indices.
-
-[float]
-=== Elasticsearch migration plugin
-
-We have provided the https://github.com/elastic/elasticsearch-migration[Elasticsearch migration plugin]
-to help you detect any issues that you may have when upgrading to
-Elasticsearch 2.0. Please install and run the plugin *before* upgrading.
-
-[float]
-=== Also see
-
-* <<breaking_20_removed_features>>
-* <<breaking_20_network_changes>>
-* <<breaking_20_multiple_literal_data_path_literal_striping>>
-* <<breaking_20_mapping_changes>>
-* <<breaking_20_crud_and_routing_changes>>
-* <<breaking_20_query_dsl_changes>>
-* <<breaking_20_search_changes>>
-* <<breaking_20_aggregation_changes>>
-* <<breaking_20_parent_child_changes>>
-* <<breaking_20_scripting_changes>>
-* <<breaking_20_index_api_changes>>
-* <<breaking_20_snapshot_and_restore_changes>>
-* <<breaking_20_plugin_and_packaging_changes>>
-* <<breaking_20_setting_changes>>
-* <<breaking_20_stats_info_and_literal_cat_literal_changes>>
-* <<breaking_20_java_api_changes>>
-
-include::migrate_2_0/removals.asciidoc[]
-
-include::migrate_2_0/network.asciidoc[]
-
-include::migrate_2_0/striping.asciidoc[]
-
-include::migrate_2_0/mapping.asciidoc[]
-
-include::migrate_2_0/crud.asciidoc[]
-
-include::migrate_2_0/query_dsl.asciidoc[]
-
-include::migrate_2_0/search.asciidoc[]
-
-include::migrate_2_0/aggs.asciidoc[]
-
-include::migrate_2_0/parent_child.asciidoc[]
-
-include::migrate_2_0/scripting.asciidoc[]
-
-include::migrate_2_0/index_apis.asciidoc[]
-
-include::migrate_2_0/snapshot_restore.asciidoc[]
-
-include::migrate_2_0/packaging.asciidoc[]
-
-include::migrate_2_0/settings.asciidoc[]
-
-include::migrate_2_0/stats.asciidoc[]
-
-include::migrate_2_0/java.asciidoc[]
diff --git a/docs/reference/migration/migrate_2_0/aggs.asciidoc b/docs/reference/migration/migrate_2_0/aggs.asciidoc
deleted file mode 100644
index 1351b4cb4a3..00000000000
--- a/docs/reference/migration/migrate_2_0/aggs.asciidoc
+++ /dev/null
@@ -1,70 +0,0 @@
-[[breaking_20_aggregation_changes]]
-=== Aggregation changes
-
-==== Min doc count defaults to zero
-
-Both the `histogram` and `date_histogram` aggregations now have a default
-`min_doc_count` of `0` instead of `1`.
-
-==== Timezone for date field
-
-Specifying the `time_zone` parameter in queries or aggregations on fields of
-type `date` must now be either an ISO 8601 UTC offset, or a timezone id. For
-example, the value `+1:00` must now be written as `+01:00`.
-
-==== Time zones and offsets
-
-The `histogram` and the `date_histogram` aggregation now support a simplified
-`offset` option that replaces the previous `pre_offset` and `post_offset`
-rounding options. Instead of having to specify two separate offset shifts of
-the underlying buckets, the `offset` option moves the bucket boundaries in
-positive or negative direction depending on its argument.
-
-The `date_histogram` options for `pre_zone` and `post_zone` are replaced by
-the `time_zone` option. The behavior of `time_zone` is equivalent to the
-former `pre_zone` option. Setting `time_zone` to a value like "+01:00" now
-will lead to the bucket calculations being applied in the specified time zone.
-The `key` is returned as the timestamp in UTC, but the `key_as_string` is
-returned in the time zone specified.
-
-In addition to this, the `pre_zone_adjust_large_interval` is removed because
-we now always return dates and bucket keys in UTC.
-
-==== Including/excluding terms
-
-`include`/`exclude` filtering on the `terms` aggregation now uses the same
-syntax as <<regexp-syntax,regexp queries>> instead of the Java regular
-expression syntax. While simple regexps should still work, more complex ones
-might need some rewriting. Also, the `flags` parameter is no longer supported.
-
-==== Boolean fields
-
-Aggregations on `boolean` fields will now return `0` and `1` as keys, and
-`"true"` and `"false"` as string keys.  See <<migration-bool-fields>> for more
-information.
-
-
-==== Java aggregation classes
-
-The `date_histogram` aggregation now returns a `Histogram` object in the
-response, and the `DateHistogram` class has been removed.  Similarly the
-`date_range`, `ipv4_range`, and `geo_distance` aggregations all return a
-`Range` object in the response, and the `IPV4Range`, `DateRange`, and
-`GeoDistance` classes have been removed.
-
-The motivation for this is to have a single response API for the Range and
-Histogram aggregations regardless of the type of data being queried.  To
-support this some changes were made in the `MultiBucketAggregation` interface
-which applies to all bucket aggregations:
-
-* The `getKey()` method now returns `Object` instead of `String`. The actual
-  object type returned depends on the type of aggregation requested (e.g. the
-  `date_histogram` will return a `DateTime` object for this method whereas a
-  `histogram` will return a `Number`).
-* A `getKeyAsString()` method has been added to return the String
-  representation of the key.
-* All other `getKeyAsX()` methods have been removed.
-* The `getBucketAsKey(String)` methods have been removed on all aggregations
-  except the `filters` and `terms` aggregations.
-
-
diff --git a/docs/reference/migration/migrate_2_0/crud.asciidoc b/docs/reference/migration/migrate_2_0/crud.asciidoc
deleted file mode 100644
index ef3ba93e67e..00000000000
--- a/docs/reference/migration/migrate_2_0/crud.asciidoc
+++ /dev/null
@@ -1,130 +0,0 @@
-[[breaking_20_crud_and_routing_changes]]
-=== CRUD and routing changes
-
-==== Explicit custom routing
-
-Custom `routing` values can no longer be extracted from the document body, but
-must be specified explicitly as part of the query string, or in the metadata
-line in the <<docs-bulk,`bulk`>> API.  See <<migration-meta-fields>> for an
-example.
-
-==== Routing hash function
-
-The default hash function that is used for routing has been changed from
-`djb2` to `murmur3`. This change should be transparent unless you relied on
-very specific properties of `djb2`. This will help ensure a better balance of
-the document counts between shards.
-
-In addition, the following routing-related node settings have been deprecated:
-
-`cluster.routing.operation.hash.type`::
-
-  This was an undocumented setting that allowed to configure which hash function
-  to use for routing. `murmur3` is now enforced on new indices.
-
-`cluster.routing.operation.use_type`::
-
-  This was an undocumented setting that allowed to take the `_type` of the
-  document into account when computing its shard (default: `false`). `false` is
-  now enforced on new indices.
-
-==== Delete API with custom routing
-
-The delete API used to be broadcast to all shards in the index which meant
-that, when using custom routing, the `routing` parameter was optional. Now,
-the delete request is forwarded only to the shard holding the document. If you
-are using custom routing then you should specify the `routing` value when
-deleting a document, just as is already required for the `index`, `create`,
-and `update` APIs.
-
-To make sure that you never forget a routing value, make routing required with
-the following mapping:
-
-[source,js]
----------------------------
-PUT my_index
-{
-  "mappings": {
-    "my_type": {
-      "_routing": {
-        "required": true
-      }
-    }
-  }
-}
----------------------------
-
-==== All stored meta-fields returned by default
-
-Previously, meta-fields like `_routing`, `_timestamp`, etc would only be
-included in a GET request if specifically requested with the `fields`
-parameter.  Now, all meta-fields which have stored values will be returned by
-default.  Additionally, they are now returned at the top level (along with
-`_index`, `_type`, and `_id`) instead of in the `fields` element.
-
-For instance, the following request:
-
-[source,sh]
----------------
-GET /my_index/my_type/1
----------------
-
-might return:
-
-[source,js]
----------------
-{
-  "_index":     "my_index",
-  "_type":      "my_type",
-  "_id":        "1",
-  "_timestamp": 10000000, <1>
-  "_source": {
-    "foo" : [ "bar" ]
-  }
-}
----------------
-<1> The `_timestamp` is returned by default, and at the top level.
-
-
-==== Async replication
-
-The `replication` parameter has been removed from all CRUD operations
-(`index`, `create`,  `update`, `delete`, `bulk`) as it interfered with the
-<<indices-synced-flush,synced flush>> feature.  These operations are now
-synchronous only and a request will only return once the changes have been
-replicated to all active shards in the shard group.
-
-Instead, use more client processes to send more requests in parallel.
-
-==== Documents must be specified without a type wrapper
-
-Previously, the document body could be wrapped in another object with the name
-of the `type`:
-
-[source,js]
---------------------------
-PUT my_index/my_type/1
-{
-  "my_type": { <1>
-    "text": "quick brown fox"
-  }
-}
---------------------------
-<1> This `my_type` wrapper is not part of the document itself, but represents the document type.
-
-This feature was deprecated before but could be reenabled with the
-`mapping.allow_type_wrapper` index setting.  This setting is no longer
-supported.  The above document should be indexed as follows:
-
-[source,js]
---------------------------
-PUT my_index/my_type/1
-{
-  "text": "quick brown fox"
-}
---------------------------
-
-==== Term Vectors API
-
-Usage of `/_termvector` is deprecated in favor of `/_termvectors`.
-
diff --git a/docs/reference/migration/migrate_2_0/index_apis.asciidoc b/docs/reference/migration/migrate_2_0/index_apis.asciidoc
deleted file mode 100644
index c177a887866..00000000000
--- a/docs/reference/migration/migrate_2_0/index_apis.asciidoc
+++ /dev/null
@@ -1,43 +0,0 @@
-[[breaking_20_index_api_changes]]
-=== Index API changes
-
-==== Index aliases
-
-
-Fields used in alias filters no longer have to exist in the mapping at alias
-creation time. Previously, alias filters were parsed at alias creation time
-and the parsed form was cached in memory. Now, alias filters are  parsed at
-request time and the fields in filters are resolved from the current mapping.
-
-This also means that index aliases now support `has_parent` and `has_child`
-queries.
-
-The <<alias-retrieving, GET alias api>> will now throw an exception if no
-matching aliases are found. This change brings the defaults for this API in
-line with the other Indices APIs. The <<multi-index>> options can be used on a
-request to change this behavior.
-
-==== File based index templates
-
-Index templates can no longer be configured on disk. Use the
-<<indices-templates,`_template`>> API instead.
-
-==== Analyze API changes
-
-
-The Analyze API now returns the `position` of the first token as `0`
-instead of `1`.
-
-The `prefer_local` parameter has been removed. The `_analyze` API is a light
-operation and the caller shouldn't be concerned about whether it executes on
-the node that receives the request or another node.
-
-The `text()` method on `AnalyzeRequest` now returns `String[]` instead of
-`String`.
-
-==== Removed `id_cache` from clear cache api
-
-The <<indices-clearcache,clear cache>> API no longer supports the `id_cache`
-option.  Instead, use the `fielddata` option to clear the cache for the
-`_parent` field.
-
diff --git a/docs/reference/migration/migrate_2_0/java.asciidoc b/docs/reference/migration/migrate_2_0/java.asciidoc
deleted file mode 100644
index b2f5ee63e0d..00000000000
--- a/docs/reference/migration/migrate_2_0/java.asciidoc
+++ /dev/null
@@ -1,147 +0,0 @@
-[[breaking_20_java_api_changes]]
-=== Java API changes
-
-==== Transport API construction
-
-The `TransportClient` construction code has changed, it now uses the builder
-pattern. Instead of:
-
-[source,java]
---------------------------------------------------
-Settings settings = Settings.settingsBuilder()
-        .put("cluster.name", "myClusterName").build();
-Client client = new TransportClient(settings);
---------------------------------------------------
-
-Use the following:
-
-[source,java]
---------------------------------------------------
-Settings settings = Settings.settingsBuilder()
-        .put("cluster.name", "myClusterName").build();
-Client client = TransportClient.builder().settings(settings).build();
---------------------------------------------------
-
-The transport client also no longer supports loading settings from config files.
-If you have a config file, you can load it into settings yourself before
-constructing the transport client:
-
-[source,java]
---------------------------------------------------
-Settings settings = Settings.settingsBuilder()
-        .loadFromPath(pathToYourSettingsFile).build();
-Client client = TransportClient.builder().settings(settings).build();
---------------------------------------------------
-
-==== Exception are only thrown on total failure
-
-Previously, many APIs would throw an exception if any shard failed to execute
-the request. Now the exception is only thrown if all shards fail the request.
-The responses for these APIs will always have a `getShardFailures` method that
-you can and should check for failures.
-
-
-==== IndexMissingException removed.
-
-Use `IndexNotFoundException` instead.
-
-
-==== Automatically thread client listeners
-
-Previously, the user had to set request listener threads to `true` when on the
-client side in order not to block IO threads on heavy operations. This proved
-to be very trappy for users, and ended up creating problems that are very hard
-to debug.
-
-In 2.0, Elasticsearch automatically threads listeners that are used from the
-client when the client is a node client or a transport client. Threading can
-no longer be manually set.
-
-
-==== Query/filter refactoring
-
-`org.elasticsearch.index.queries.FilterBuilders` has been removed as part of the merge of
-queries and filters. These filters are now available in `QueryBuilders` with the same name.
-All methods that used to accept a `FilterBuilder` now accept a `QueryBuilder` instead.
-
-In addition some query builders have been removed or renamed:
-
-* `commonTerms(...)` renamed with `commonTermsQuery(...)`
-* `queryString(...)` renamed with `queryStringQuery(...)`
-* `simpleQueryString(...)` renamed with `simpleQueryStringQuery(...)`
-* `textPhrase(...)` removed
-* `textPhrasePrefix(...)` removed
-* `textPhrasePrefixQuery(...)` removed
-* `filtered(...)` removed. Use `filteredQuery(...)` instead.
-* `inQuery(...)` removed.
-
-==== GetIndexRequest
-
-`GetIndexRequest.features()` now returns an array of Feature Enums instead of an array of String values.
-
-The following deprecated methods have been removed:
-
-* `GetIndexRequest.addFeatures(String[])` - Use
-  `GetIndexRequest.addFeatures(Feature[])` instead
-
-* `GetIndexRequest.features(String[])` - Use
-  `GetIndexRequest.features(Feature[])` instead.
-
-* `GetIndexRequestBuilder.addFeatures(String[])` - Use
-  `GetIndexRequestBuilder.addFeatures(Feature[])` instead.
-
-* `GetIndexRequestBuilder.setFeatures(String[])` - Use
-  `GetIndexRequestBuilder.setFeatures(Feature[])` instead.
-
-
-==== BytesQueryBuilder removed
-
-The redundant BytesQueryBuilder has been removed in favour of the
-WrapperQueryBuilder internally.
-
-==== TermsQueryBuilder execution removed
-
-The `TermsQueryBuilder#execution` method has been removed as it has no effect, it is ignored by the
- corresponding parser.
-
-==== ImmutableSettings removed
-
-Use `Settings.builder()` instead of `ImmutableSettings.builder()`.
-
-==== InetSocketTransportAddress removed
-
-Use `InetSocketTransportAddress(InetSocketAddress address)` instead of `InetSocketTransportAddress(String, int)`.
-You can create an InetSocketAddress instance with `InetSocketAddress(String, int)`. For example:
-
-[source,java]
------------------------------
-new InetSocketTransportAddress(new InetSocketAddress("127.0.0.1", 0));
------------------------------
-
-==== Request Builders refactoring
-
-An `action` parameter has been added to various request builders:
-
-* Instead of `new SnapshotsStatusRequestBuilder(elasticSearchClient)` use `new SnapshotsStatusRequestBuilder(elasticSearchClient, SnapshotsStatusAction.INSTANCE)`.
-
-* Instead of `new CreateSnapshotRequestBuilder(elasticSearchClient)` use `new CreateSnapshotRequestBuilder(elasticSearchClient, CreateSnapshotAction.INSTANCE)`.
-
-* Instead of `new CreateIndexRequestBuilder(elasticSearchClient, index)` use `new CreateIndexRequestBuilder(elasticSearchClient, CreateIndexAction.INSTANCE, index)`.
-
-==== Shading and package relocation removed
-
-Elasticsearch used to shade its dependencies and to relocate packages. We no longer use shading or relocation.
-You might need to change your imports to the original package names:
-
-* `com.google.common` was `org.elasticsearch.common`
-* `com.carrotsearch.hppc` was `org.elasticsearch.common.hppc`
-* `jsr166e` was `org.elasticsearch.common.util.concurrent.jsr166e`
-* `com.fasterxml.jackson` was `org.elasticsearch.common.jackson`
-* `org.joda.time` was `org.elasticsearch.common.joda.time`
-* `org.joda.convert` was `org.elasticsearch.common.joda.convert`
-* `org.jboss.netty` was `org.elasticsearch.common.netty`
-* `com.ning.compress` was `org.elasticsearch.common.compress`
-* `com.github.mustachejava` was `org.elasticsearch.common.mustache`
-* `com.tdunning.math.stats` was `org.elasticsearch.common.stats`
-* `org.apache.commons.lang` was `org.elasticsearch.common.lang`
-* `org.apache.commons.cli` was `org.elasticsearch.common.cli.commons`
diff --git a/docs/reference/migration/migrate_2_0/mapping.asciidoc b/docs/reference/migration/migrate_2_0/mapping.asciidoc
deleted file mode 100644
index b4ee0d54412..00000000000
--- a/docs/reference/migration/migrate_2_0/mapping.asciidoc
+++ /dev/null
@@ -1,439 +0,0 @@
-[[breaking_20_mapping_changes]]
-=== Mapping changes
-
-A number of changes have been made to mappings to remove ambiguity and to
-ensure that conflicting mappings cannot be created.
-
-One major change is that dynamically added fields must have their mapping
-confirmed by the master node before indexing continues.  This is to avoid a
-problem where different shards in the same index dynamically add different
-mappings for the same field.  These conflicting mappings can silently return
-incorrect results and can lead to index corruption.
-
-This change can make indexing slower when frequently adding many new fields.
-We are looking at ways of optimising this process but we chose safety over
-performance for this extreme use case.
-
-==== Conflicting field mappings
-
-Fields with the same name, in the same index, in different types, must have
-the same mapping, with the exception of the <<copy-to>>, <<dynamic>>,
-<<enabled>>, <<ignore-above>>, <<include-in-all>>, and <<properties>>
-parameters, which may have different settings per field.
-
-[source,js]
----------------
-PUT my_index
-{
-  "mappings": {
-    "type_one": {
-      "properties": {
-        "name": { <1>
-          "type": "string"
-        }
-      }
-    },
-    "type_two": {
-      "properties": {
-        "name": { <1>
-          "type":     "string",
-          "analyzer": "english"
-        }
-      }
-    }
-  }
-}
----------------
-<1> The two `name` fields have conflicting mappings and will prevent Elasticsearch
-    from starting.
-
-Elasticsearch will not start in the presence of conflicting field mappings.
-These indices must be deleted or reindexed using a new mapping.
-
-The `ignore_conflicts` option of the put mappings API has been removed.
-Conflicts can't be ignored anymore.
-
-==== Fields cannot be referenced by short name
-
-A field can no longer be referenced using its short name.  Instead, the full
-path to the field is required.  For instance:
-
-[source,js]
----------------
-PUT my_index
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "title":     { "type": "string" }, <1>
-        "name": {
-          "properties": {
-            "title": { "type": "string" }, <2>
-            "first": { "type": "string" },
-            "last":  { "type": "string" }
-          }
-        }
-      }
-    }
-  }
-}
----------------
-<1> This field is referred to as `title`.
-<2> This field is referred to as `name.title`.
-
-Previously, the two `title` fields in the example above could have been
-confused with each other when using the short name `title`.
-
-==== Type name prefix removed
-
-Previously, two fields with the same name in two different types could
-sometimes be disambiguated by prepending the type name.  As a side effect, it
-would add a filter on the type name to the relevant query.  This feature was
-ambiguous -- a type name could be confused with a field name -- and didn't
-work everywhere e.g. aggregations.
-
-Instead, fields should be specified with the full path, but without a type
-name prefix.  If you wish to filter by the `_type` field, either specify the
-type in the URL or add an explicit filter.
-
-The following example query in 1.x:
-
-[source,js]
-----------------------------
-GET my_index/_search
-{
-  "query": {
-    "match": {
-      "my_type.some_field": "quick brown fox"
-    }
-  }
-}
-----------------------------
-
-would be rewritten in 2.0 as:
-
-[source,js]
-----------------------------
-GET my_index/my_type/_search <1>
-{
-  "query": {
-    "match": {
-      "some_field": "quick brown fox" <2>
-    }
-  }
-}
-----------------------------
-<1> The type name can be specified in the URL to act as a filter.
-<2> The field name should be specified without the type prefix.
-
-==== Field names may not contain dots
-
-In 1.x, it was possible to create fields with dots in their name, for
-instance:
-
-[source,js]
-----------------------------
-PUT my_index
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "foo.bar": { <1>
-          "type": "string"
-        },
-        "foo": {
-          "properties": {
-            "bar": { <1>
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  }
-}
-----------------------------
-<1> These two fields cannot be distinguished as both are referred to as `foo.bar`.
-
-You can no longer create fields with dots in the name.
-
-==== Type names may not start with a dot
-
-In 1.x, Elasticsearch would issue a warning if a type name included a dot,
-e.g. `my.type`.  Now that type names are no longer used to distinguish between
-fields in different types, this warning has been relaxed: type names may now
-contain dots, but they may not *begin* with a dot.  The only exception to this
-is the special `.percolator` type.
-
-==== Type names may not be longer than 255 characters
-
-Mapping type names may not be longer than 255 characters.  Long type names
-will continue to function on indices created before upgrade, but it will not
-be possible create types with long names in new indices.
-
-==== Types may no longer be deleted
-
-In 1.x it was possible to delete a type mapping, along with all of the
-documents of that type, using the delete mapping API.  This is no longer
-supported, because remnants of the fields in the type could remain in the
-index, causing corruption later on.
-
-Instead, if you need to delete a type mapping, you should reindex to a new
-index which does not contain the mapping.  If you just need to delete the
-documents that belong to that type, then use the delete-by-query plugin
-instead.
-
-[[migration-meta-fields]]
-==== Type meta-fields
-
-The <<mapping-fields,meta-fields>> associated with had configuration options
-removed, to make them more reliable:
-
-* `_id` configuration can no longer be changed.  If you need to sort, use the <<mapping-uid-field,`_uid`>> field instead.
-* `_type` configuration can no longer be changed.
-* `_index` configuration can no longer be changed.
-* `_routing` configuration is limited to marking routing as required.
-* `_field_names` configuration is limited to disabling the field.
-* `_size` configuration is limited to enabling the field.
-* `_timestamp` configuration is limited to enabling the field, setting format and default value.
-* `_boost` has been removed.
-* `_analyzer` has been removed.
-
-Importantly, *meta-fields can no longer be specified as part of the document
-body.*  Instead, they must be specified in the query string parameters.  For
-instance, in 1.x, the `routing` could be specified as follows:
-
-[source,json]
------------------------------
-PUT my_index
-{
-  "mappings": {
-    "my_type": {
-      "_routing": {
-        "path": "group" <1>
-      },
-      "properties": {
-        "group": { <1>
-          "type": "string"
-        }
-      }
-    }
-  }
-}
-
-PUT my_index/my_type/1 <2>
-{
-  "group": "foo"
-}
------------------------------
-<1> This 1.x mapping tells Elasticsearch to extract the `routing` value from the `group` field in the document body.
-<2> This indexing request uses a `routing` value of `foo`.
-
-In 2.0, the routing must be specified explicitly:
-
-[source,json]
------------------------------
-PUT my_index
-{
-  "mappings": {
-    "my_type": {
-      "_routing": {
-        "required": true <1>
-      },
-      "properties": {
-        "group": {
-          "type": "string"
-        }
-      }
-    }
-  }
-}
-
-PUT my_index/my_type/1?routing=bar <2>
-{
-  "group": "foo"
-}
------------------------------
-<1> Routing can be marked as required to ensure it is not forgotten during indexing.
-<2> This indexing request uses a `routing` value of `bar`.
-
-==== `_timestamp` and `_ttl` deprecated
-
-The `_timestamp` and `_ttl` fields are deprecated, but will remain functional
-for the remainder of the 2.x series.
-
-Instead of the `_timestamp` field, use a normal <<date,`date`>> field and set
-the value explicitly.
-
-The current `_ttl` functionality will be replaced in a future version with a
-new implementation of TTL, possibly with different semantics, and will not
-depend on the `_timestamp` field.
-
-==== Analyzer mappings
-
-Previously, `index_analyzer` and `search_analyzer` could be set separately,
-while the `analyzer` setting would set both.  The `index_analyzer` setting has
-been removed in favour of just using the `analyzer` setting.
-
-If just the `analyzer` is set, it will be used at index time and at search time.  To use a different analyzer at search time, specify both the `analyzer` and a `search_analyzer`.
-
-The `index_analyzer`, `search_analyzer`,  and `analyzer` type-level settings
-have also been removed, as it is no longer possible to select fields based on
-the type name.
-
-The `_analyzer` meta-field, which allowed setting an analyzer per document has
-also been removed.  It will be ignored on older indices.
-
-==== Date fields and Unix timestamps
-
-Previously, `date` fields would first try to parse values as a Unix timestamp
--- milliseconds-since-the-epoch -- before trying to use their defined date
-`format`.  This meant that formats like `yyyyMMdd` could never work, as values
-would be interpreted as timestamps.
-
-In 2.0, we have added two formats: `epoch_millis` and `epoch_second`.  Only
-date fields that use these formats will be able to parse timestamps.
-
-These formats cannot be used in dynamic templates, because they are
-indistinguishable from long values.
-
-==== Default date format
-
-The default date format has changed from `date_optional_time` to
-`strict_date_optional_time`, which expects a 4 digit year, and a 2 digit month
-and day, (and optionally, 2 digit hour, minute, and second).
-
-A dynamically added date field, by default, includes the `epoch_millis`
-format to support timestamp parsing.  For instance:
-
-[source,js]
--------------------------
-PUT my_index/my_type/1
-{
-  "date_one": "2015-01-01" <1>
-}
--------------------------
-<1> Has `format`: `"strict_date_optional_time||epoch_millis"`.
-
-==== `mapping.date.round_ceil` setting
-
-The `mapping.date.round_ceil` setting for date math parsing has been removed.
-
-[[migration-bool-fields]]
-==== Boolean fields
-
-Boolean fields used to have a string fielddata with `F` meaning `false` and `T`
-meaning `true`. They have been refactored to use numeric fielddata, with `0`
-for `false` and `1` for `true`. As a consequence, the format of the responses of
-the following APIs changed when applied to boolean fields: `0`/`1` is returned
-instead of `F`/`T`:
-
-* <<search-request-fielddata-fields,fielddata fields>>
-* <<search-request-sort,sort values>>
-* <<search-aggregations-bucket-terms-aggregation,terms aggregations>>
-
-In addition, terms aggregations use a custom formatter for boolean (like for
-dates and ip addresses, which are also backed by numbers) in order to return
-the user-friendly representation of boolean fields: `false`/`true`:
-
-[source,js]
----------------
-"buckets": [
-  {
-     "key": 0,
-     "key_as_string": "false",
-     "doc_count": 42
-  },
-  {
-     "key": 1,
-     "key_as_string": "true",
-     "doc_count": 12
-  }
-]
----------------
-
-==== `index_name` and `path` removed
-
-The `index_name` setting was used to change the name of the Lucene field,
-and the `path` setting was used on `object` fields to determine whether the
-Lucene field should use the full path (including parent object fields), or
-just the final `name`.
-
-These setting have been removed as their purpose is better served with the
-<<copy-to>> parameter.
-
-==== Murmur3 Fields
-
-Fields of type `murmur3` can no longer change `doc_values` or `index` setting.
-They are always mapped as follows:
-
-[source,js]
----------------------
-{
-  "type":       "murmur3",
-  "index":      "no",
-  "doc_values": true
-}
----------------------
-
-==== Mappings in config files not supported
-
-The ability to specify mappings in configuration files has been removed. To
-specify default mappings that apply to multiple indexes, use
-<<indices-templates,index templates>> instead.
-
-Along with this change, the following settings have been removed:
-
-* `index.mapper.default_mapping_location`
-* `index.mapper.default_percolator_mapping_location`
-
-==== Fielddata formats
-
-Now that doc values are the default for fielddata, specialized in-memory
-formats have become an esoteric option. These fielddata formats have been removed:
-
-* `fst` on string fields
-* `compressed` on geo points
-
-The default fielddata format will be used instead.
-
-==== Posting and doc-values codecs
-
-It is no longer possible to specify per-field postings and doc values formats
-in the mappings. This setting will be ignored on indices created before 2.0
-and will cause mapping parsing to fail on indices created on or after 2.0. For
-old indices, this means that new segments will be written with the default
-postings and doc values formats of the current codec.
-
-It is still possible to change the whole codec by using the `index.codec`
-setting. Please however note that using a non-default codec is discouraged as
-it could prevent future versions of Elasticsearch from being able to read the
-index.
-
-==== Compress and compress threshold
-
-The `compress` and `compress_threshold` options have been removed from the
-`_source` field and fields of type `binary`.  These fields are compressed by
-default.  If you would like to increase compression levels, use the new
-<<index-codec,`index.codec: best_compression`>> setting instead.
-
-==== position_offset_gap
-
-The `position_offset_gap` option is renamed to 'position_increment_gap'. This was
-done to clear away the confusion. Elasticsearch's 'position_increment_gap' now is
-mapped directly to Lucene's 'position_increment_gap'
-
-The default `position_increment_gap` is now 100. Indexes created in Elasticsearch
-2.0.0 will default to using 100 and indexes created before that will continue
-to use the old default of 0. This was done to prevent phrase queries from
-matching across different values of the same term unexpectedly. Specifically,
-100 was chosen to cause phrase queries with slops up to 99 to match only within
-a single value of a field.
-
-==== copy_to and multi fields
-
-A <<copy-to,copy_to>> within a <<multi-fields,multi field>> is ignored from version 2.0 on. With any version after
-2.1 or 2.0.1 creating a mapping that has a copy_to within a multi field will result 
-in an exception.
-
-
diff --git a/docs/reference/migration/migrate_2_0/network.asciidoc b/docs/reference/migration/migrate_2_0/network.asciidoc
deleted file mode 100644
index d493bff5688..00000000000
--- a/docs/reference/migration/migrate_2_0/network.asciidoc
+++ /dev/null
@@ -1,39 +0,0 @@
-[[breaking_20_network_changes]]
-=== Network changes
-
-==== Bind to localhost
-
-Elasticsearch 2.x will only bind to localhost by default. It will try to bind
-to both 127.0.0.1 (IPv4) and [::1] (IPv6), but will work happily in
-environments where only IPv4 or IPv6 is available. This change prevents
-Elasticsearch from trying to connect to other nodes on your network unless you
-specifically tell it to do so. When moving to production you should configure
-the `network.host` parameter, either in the `elasticsearch.yml` config file or
-on the command line:
-
-[source,sh]
---------------------
-bin/elasticsearch --network.host 192.168.1.5
-bin/elasticsearch --network.host _non_loopback_
---------------------
-
-The full list of options that network.host accepts can be found in the <<modules-network>>.
-
-==== Unicast discovery
-
-When bound to localhost, Elasticsearch will use unicast to contact
-the first 5 ports in the `transport.tcp.port` range, which defaults to
-`9300-9400`. This preserves the zero-config auto-clustering experience for the developer,
-but it means that you will have to provide a list of <<unicast,unicast hosts>>
-when moving to production, for instance:
-
-[source,yaml]
----------------------
-discovery.zen.ping.unicast.hosts: [ 192.168.1.2,  192.168.1.3 ]
----------------------
-
-You don’t need to list all of the nodes in your cluster as unicast hosts, but
-you should specify at least a quorum (majority) of master-eligible nodes. A
-big cluster will typically have three dedicated master nodes, in which case we
-recommend listing all three of them as unicast hosts.
-
diff --git a/docs/reference/migration/migrate_2_0/packaging.asciidoc b/docs/reference/migration/migrate_2_0/packaging.asciidoc
deleted file mode 100644
index dae87187ba4..00000000000
--- a/docs/reference/migration/migrate_2_0/packaging.asciidoc
+++ /dev/null
@@ -1,84 +0,0 @@
-[[breaking_20_plugin_and_packaging_changes]]
-=== Plugin and packaging changes
-
-==== Symbolic links and paths
-
-Elasticsearch 2.0 runs with the Java security manager enabled and is much more
-restrictive about which paths it is allowed to access.  Various paths can be
-configured, e.g. `path.data`, `path.scripts`, `path.repo`.  A configured path
-may itself be a symbolic link, but no symlinks under that path will be
-followed.
-
-==== Running `bin/elasticsearch`
-
-The command line parameter parsing has been rewritten to deal properly with
-spaces in parameters. All config settings can still be specified on the
-command line when starting Elasticsearch, but they must appear after the
-built-in "static parameters", such as `-d` (to daemonize) and `-p` (the PID path).
-
-For instance:
-
-[source,sh]
------------
-bin/elasticsearch -d -p /tmp/foo.pid --http.cors.enabled=true --http.cors.allow-origin='*'
------------
-
-For a list of static parameters, run `bin/elasticsearch -h`
-
-==== `-f` removed
-
-The `-f` parameter, which used to indicate that Elasticsearch should be run in
-the foreground, was deprecated in 1.0 and removed in 2.0.
-
-==== `V` for version
-
-The `-v` parameter now means `--verbose` for both `bin/elasticsearch-plugin` and
-`bin/elasticsearch` (although it has no effect on the latter).  To output the
-version, use `-V` or `--version` instead.
-
-==== Plugin manager should run as root
-
-The permissions of the `config`, `bin`, and `plugins` directories in the RPM
-and deb packages have been made more restrictive.  The plugin manager should
-be run as root otherwise it will not be able to install plugins.
-
-==== Support for official plugins
-
-Almost all of the official Elasticsearch plugins have been moved to the main
-`elasticsearch` repository. They will be released at the same time as
-Elasticsearch and have the same version number as Elasticsearch.
-
-Official plugins can be installed as follows:
-
-[source,sh]
----------------
-sudo bin/elasticsearch-plugin install analysis-icu
----------------
-
-Community-provided plugins can be installed as before.
-
-==== Plugins require descriptor file
-
-All plugins are now required to have a https://github.com/elastic/elasticsearch/blob/2.0/dev-tools/src/main/resources/plugin-metadata/plugin-descriptor.properties[plugin-descriptor.properties]  file.  If a node has a plugin installed which lacks this file, it will be unable to start.
-
-==== Repository naming structure changes
-
-Elasticsearch 2.0 changes the way the repository URLs are referenced. Instead
-of specific repositories for both major and minor versions, the repositories will
-use a major version reference only.
-
-The URL for apt packages now uses the following structure;
-
-[source,sh]
----------------
-deb http://packages.elastic.co/elasticsearch/2.x/debian stable main
----------------
-
-And for yum packages it is;
-
-[source,sh]
----------------
-baseurl=http://packages.elastic.co/elasticsearch/2.x/centos
----------------
-
-The <<setup-repositories, repositories>> page details this change.
diff --git a/docs/reference/migration/migrate_2_0/parent_child.asciidoc b/docs/reference/migration/migrate_2_0/parent_child.asciidoc
deleted file mode 100644
index 1addf883973..00000000000
--- a/docs/reference/migration/migrate_2_0/parent_child.asciidoc
+++ /dev/null
@@ -1,43 +0,0 @@
-[[breaking_20_parent_child_changes]]
-=== Parent/Child changes
-
-Parent/child has been rewritten completely to reduce memory usage and to
-execute `has_child` and `has_parent` queries faster and more efficient. The
-`_parent` field uses doc values by default. The refactored and improved
-implementation is only active for indices created on or after version 2.0.
-
-In order to benefit from all the performance and memory improvements, we
-recommend reindexing all existing indices that use the `_parent` field.
-
-==== Parent type cannot pre-exist
-
-A mapping type is declared as a child of another mapping type by specifying
-the `_parent` meta field:
-
-[source,js]
---------------------------
-DELETE *
-
-PUT my_index
-{
-  "mappings": {
-    "my_parent": {},
-    "my_child": {
-      "_parent": {
-        "type": "my_parent" <1>
-      }
-    }
-  }
-}
---------------------------
-<1> The `my_parent` type is the parent of the `my_child` type.
-
-The mapping for the parent type can be added at the same time as the mapping
-for the child type, but cannot be added before the child type.
-
-==== `top_children` query removed
-
-The `top_children` query has been removed in favour of the `has_child` query.
-It wasn't always faster than the `has_child` query and the results were usually
-inaccurate. The total hits and any aggregations in the same search request
-would be incorrect if `top_children` was used.
diff --git a/docs/reference/migration/migrate_2_0/query_dsl.asciidoc b/docs/reference/migration/migrate_2_0/query_dsl.asciidoc
deleted file mode 100644
index a85ade8690b..00000000000
--- a/docs/reference/migration/migrate_2_0/query_dsl.asciidoc
+++ /dev/null
@@ -1,189 +0,0 @@
-[[breaking_20_query_dsl_changes]]
-=== Query DSL changes
-
-==== Queries and filters merged
-
-Queries and filters have been merged -- all filter clauses are now query
-clauses. Instead, query clauses can now be used in _query context_ or in
-_filter context_:
-
-Query context::
-
-A query used in query context will calculate relevance scores and will not be
-cacheable.  Query context is used whenever filter context does not apply.
-
-Filter context::
-+
---
-
-A query used in filter context will not calculate relevance scores, and will
-be cacheable. Filter context is introduced by:
-
-* the `constant_score` query
-* the `must_not` and (newly added) `filter` parameter in the `bool` query
-* the `filter` and `filters` parameters in the `function_score` query
-* any API called `filter`, such as the `post_filter` search parameter, or in
-  aggregations or index aliases
---
-
-==== `terms` query and filter
-
-The `execution` option of the `terms` filter is now deprecated and is ignored
-if provided. Similarly, the `terms` query no longer supports the
-`minimum_should_match` parameter.
-
-==== `or` and `and` now implemented via `bool`
-
-The `or` and `and` filters previously had a different execution pattern to the
-`bool` filter. It used to be important to use `and`/`or` with certain filter
-clauses, and `bool` with others.
-
-This distinction has been removed: the `bool` query is now smart enough to
-handle both cases optimally.  As a result of this change, the `or` and `and`
-filters are now sugar syntax which are executed internally as a `bool` query.
-These filters may be removed in the future.
-
-==== `filtered` query and `query` filter deprecated
-
-The `query` filter is deprecated as is it no longer needed -- all queries can
-be used in query or filter context.
-
-The `filtered` query is deprecated in favour of the `bool` query. Instead of
-the following:
-
-[source,js]
--------------------------
-GET _search
-{
-  "query": {
-    "filtered": {
-      "query": {
-        "match": {
-          "text": "quick brown fox"
-        }
-      },
-      "filter": {
-        "term": {
-          "status": "published"
-        }
-      }
-    }
-  }
-}
--------------------------
-
-move the query and filter to the `must` and `filter` parameters in the `bool`
-query:
-
-[source,js]
--------------------------
-GET _search
-{
-  "query": {
-    "bool": {
-      "must": {
-        "match": {
-          "text": "quick brown fox"
-        }
-      },
-      "filter": {
-        "term": {
-          "status": "published"
-        }
-      }
-    }
-  }
-}
--------------------------
-
-==== Filter auto-caching
-
-It used to be possible to control which filters were cached with the `_cache`
-option and to provide a custom `_cache_key`.  These options are deprecated
-and, if present, will be ignored.
-
-Query clauses used in filter context are now auto-cached when it makes sense
-to do so.  The algorithm takes into account the frequency of use, the cost of
-query execution, and the cost of building the filter.
-
-The `terms` filter lookup mechanism no longer caches the values of the
-document containing the terms.  It relies on the filesystem cache instead. If
-the lookup index is not too large, it is recommended to replicate it to all
-nodes by setting `index.auto_expand_replicas: 0-all` in order to remove the
-network overhead as well.
-
-==== Numeric queries use IDF for scoring
-
-Previously, term queries on numeric fields were deliberately prevented from
-using the usual Lucene scoring logic and this behaviour was undocumented and,
-to some, unexpected.
-
-Single `term` queries on numeric fields now score in the same way as string
-fields, using IDF and norms (if enabled).
-
-To query numeric fields without scoring, the query clause should be used in
-filter context, e.g. in the `filter` parameter of the `bool` query, or wrapped
-in a `constant_score` query:
-
-[source,js]
-----------------------------
-GET _search
-{
-  "query": {
-    "bool": {
-      "must": [
-        {
-          "match": { <1>
-            "numeric_tag": 5
-          }
-        }
-      ],
-      "filter": [
-        {
-          "match": { <2>
-            "count": 5
-          }
-        }
-      ]
-    }
-  }
-}
-----------------------------
-<1> This clause would include IDF in the relevance score calculation.
-<2> This clause would have no effect on the relevance score.
-
-==== Fuzziness and fuzzy-like-this
-
-Fuzzy matching used to calculate the score for each fuzzy alternative, meaning
-that rare misspellings would have a higher score than the more common correct
-spellings. Now, fuzzy matching blends the scores of all the fuzzy alternatives
-to use the IDF of the most frequently occurring alternative.
-
-Fuzziness can no longer be specified using a percentage, but should instead
-use the number of allowed edits:
-
-* `0`, `1`, `2`, or
-* `AUTO` (which chooses `0`, `1`, or `2` based on the length of the term)
-
-The `fuzzy_like_this` and `fuzzy_like_this_field` queries used a very
-expensive approach to fuzzy matching and have been removed.
-
-==== More Like This
-
-The More Like This (`mlt`) API and the `more_like_this_field` (`mlt_field`)
-query have been removed in favor of the
-<<query-dsl-mlt-query, `more_like_this`>> query.
-
-The parameter `percent_terms_to_match` has been removed in favor of
-`minimum_should_match`.
-
-==== `limit` filter deprecated
-
-The `limit` filter is deprecated and becomes a no-op. You can achieve similar
-behaviour using the <<search-request-body,terminate_after>> parameter.
-
-==== Java plugins registering custom queries
-
-Java plugins that register custom queries can do so by using the
-`IndicesQueriesModule#addQuery(Class<? extends QueryParser>)` method. Other
-ways to register custom queries are not supported anymore.
diff --git a/docs/reference/migration/migrate_2_0/removals.asciidoc b/docs/reference/migration/migrate_2_0/removals.asciidoc
deleted file mode 100644
index 31693c3d3ac..00000000000
--- a/docs/reference/migration/migrate_2_0/removals.asciidoc
+++ /dev/null
@@ -1,100 +0,0 @@
-[[breaking_20_removed_features]]
-=== Removed features
-
-==== Rivers have been removed
-
-Elasticsearch does not support rivers anymore. While we had first planned to
-keep them around to ease migration, keeping support for rivers proved to be
-challenging as it conflicted with other important changes that we wanted to
-bring to 2.0 like synchronous dynamic mappings updates, so we eventually
-decided to remove them entirely. See
-link:/blog/deprecating_rivers[Deprecating Rivers] for more background about
-why we took this decision.
-
-==== Facets have been removed
-
-Facets, deprecated since 1.0, have now been removed.  Instead, use the much
-more powerful and flexible <<search-aggregations,aggregations>> framework.
-This also means that Kibana 3 will not work with Elasticsearch 2.0.
-
-==== MVEL has been removed
-
-The MVEL scripting language has been removed.  The default scripting language
-is now Groovy.
-
-==== Delete-by-query is now a plugin
-
-The old delete-by-query functionality was fast but unsafe.  It could lead to
-document differences between the primary and replica shards, and could even
-produce out of memory exceptions and cause the cluster to crash.
-
-This feature has been reimplemented using the <<search-request-scroll,scroll>> and
-<<docs-bulk,`bulk`>> APIs, which may be slower for queries which match
-large numbers of documents, but is safe.
-
-Currently, a long running delete-by-query job cannot be cancelled, which is
-one of the reasons that this functionality is only available as a plugin.  You
-can install the plugin with:
-
-[source,sh]
-------------------
-./bin/elasticsearch-plugin install delete-by-query
-------------------
-
-See {plugins}/plugins-delete-by-query.html for more information.
-
-==== Multicast Discovery is now a plugin
-
-Support for multicast is very patchy. Linux doesn’t allow multicast listening on localhost,
-while OS/X sends multicast broadcasts across all interfaces regardless of the configured
-bind address. On top of that, some networks have multicast disabled by default.
-
-This feature has been moved to a plugin. The default discovery mechanism now uses
-unicast, with a default setup which looks for the first 5 ports on localhost. If you
-still need to use multicast discovery, you can install the plugin with:
-
-[source,sh]
-------------------
-./bin/elasticsearch-plugin install discovery-multicast
-------------------
-
-==== `_shutdown` API
-
-The `_shutdown` API has been removed without a replacement. Nodes should be
-managed via the operating system and the provided start/stop scripts.
-
-==== `murmur3` is now a plugin
-
-The `murmur3` field, which indexes hashes of the field values, has been moved
-out of core and is available as a plugin. It can be installed as:
-
-[source,sh]
-------------------
-./bin/elasticsearch-plugin install mapper-murmur3
-------------------
-
-==== `_size` is now a plugin
-
-The `_size` meta-data field, which indexes the size in bytes of the original
-JSON document, has been moved out of core and is available as a plugin.  It
-can be installed as:
-
-[source,sh]
-------------------
-./bin/elasticsearch-plugin install mapper-size
-------------------
-
-==== Thrift and memcached transport
-
-The thrift and memcached transport plugins are no longer supported.  Instead, use
-either the HTTP transport (enabled by default) or the node or transport Java client.
-
-==== Bulk UDP
-
-The bulk UDP API has been removed.  Instead, use the standard
-<<docs-bulk,`bulk`>> API, or use UDP to send documents to Logstash first.
-
-==== MergeScheduler pluggability
-
-The merge scheduler is no longer pluggable.
-
diff --git a/docs/reference/migration/migrate_2_0/scripting.asciidoc b/docs/reference/migration/migrate_2_0/scripting.asciidoc
deleted file mode 100644
index 495d2daa2c5..00000000000
--- a/docs/reference/migration/migrate_2_0/scripting.asciidoc
+++ /dev/null
@@ -1,103 +0,0 @@
-[[breaking_20_scripting_changes]]
-=== Scripting changes
-
-==== Scripting syntax
-
-The syntax for scripts has been made consistent across all APIs. The accepted
-format is as follows:
-
-Inline/Dynamic scripts::
-+
---
-
-[source,js]
----------------
-"script": {
-  "inline": "doc['foo'].value + val", <1>
-  "lang":   "groovy", <2>
-  "params": { "val": 3 } <3>
-}
----------------
-<1> The inline script to execute.
-<2> The optional language of the script.
-<3> Any named parameters.
---
-
-Indexed scripts::
-+
---
-[source,js]
----------------
-"script": {
-  "id":     "my_script_id", <1>
-  "lang":   "groovy", <2>
-  "params": { "val": 3 } <3>
-}
----------------
-<1> The ID of the indexed script.
-<2> The optional language of the script.
-<3> Any named parameters.
---
-
-File scripts::
-+
---
-[source,js]
----------------
-"script": {
-  "file":   "my_file", <1>
-  "lang":   "groovy", <2>
-  "params": { "val": 3 } <3>
-}
----------------
-<1> The filename of the script, without the `.lang` suffix.
-<2> The optional language of the script.
-<3> Any named parameters.
---
-
-For example, an update request might look like this:
-
-[source,js]
----------------
-POST my_index/my_type/1/_update
-{
-  "script": {
-    "inline": "ctx._source.count += val",
-    "params": { "val": 3 }
-  },
-  "upsert": {
-    "count": 0
-  }
-}
----------------
-
-A short syntax exists for running inline scripts in the default scripting
-language without any parameters:
-
-[source,js]
-----------------
-GET _search
-{
-  "script_fields": {
-    "concat_fields": {
-      "script": "doc['one'].value + ' ' + doc['two'].value"
-    }
-  }
-}
-----------------
-
-==== Scripting settings
-
-The `script.disable_dynamic` node setting has been replaced by fine-grained
-script settings described in <<migration-script-settings>>.
-
-==== Groovy scripts sandbox
-
-The Groovy sandbox and related settings have been removed. Groovy is now a
-non-sandboxed scripting language, without any option to turn the sandbox on.
-
-==== Plugins making use of scripts
-
-Plugins that make use of scripts must register their own script context
-through `ScriptModule`. Script contexts can be used as part of fine-grained
-settings to enable/disable scripts selectively.
diff --git a/docs/reference/migration/migrate_2_0/search.asciidoc b/docs/reference/migration/migrate_2_0/search.asciidoc
deleted file mode 100644
index 036313077ff..00000000000
--- a/docs/reference/migration/migrate_2_0/search.asciidoc
+++ /dev/null
@@ -1,122 +0,0 @@
-[[breaking_20_search_changes]]
-=== Search changes
-
-==== Partial fields
-
-Partial fields have been removed in favor of <<search-request-source-filtering,source filtering>>.
-
-==== `search_type=count` deprecated
-
-The `count` search type has been deprecated. All benefits from this search
-type can now be achieved by using the (default) `query_then_fetch` search type
-and setting `size` to `0`.
-
-==== The count api internally uses the search api
-
-The count api is now a shortcut to the search api with `size` set to 0. As a
-result, a total failure will result in an exception being returned rather
-than a normal response with `count` set to `0` and shard failures.
-
-==== All stored meta-fields returned by default
-
-Previously, meta-fields like `_routing`, `_timestamp`, etc would only be
-included in the search results if specifically requested with the `fields`
-parameter.  Now, all meta-fields which have stored values will be returned by
-default.  Additionally, they are now returned at the top level (along with
-`_index`, `_type`, and `_id`) instead of in the `fields` element.
-
-For instance, the following request:
-
-[source,sh]
----------------
-GET /my_index/_search?fields=foo
----------------
-
-might return:
-
-[source,js]
----------------
-{
-   [...]
-   "hits": {
-      "total": 1,
-      "max_score": 1,
-      "hits": [
-         {
-            "_index":     "my_index",
-            "_type":      "my_type",
-            "_id":        "1",
-            "_score":     1,
-            "_timestamp": 10000000, <1>
-            "fields": {
-              "foo" : [ "bar" ]
-            }
-         }
-      ]
-   }
-}
----------------
-<1> The `_timestamp` is returned by default, and at the top level.
-
-
-==== Script fields
-
-Script fields in 1.x were only returned as a single value. Even if the return
-value of a script was a list, it would be returned as an array containing an
-array:
-
-[source,js]
----------------
-"fields": {
-  "my_field": [
-    [
-      "v1",
-      "v2"
-    ]
-  ]
-}
----------------
-
-In elasticsearch 2.0, scripts that return a list of values are treated as
-multivalued fields. The same example would return the following response, with
-values in a single array.
-
-[source,js]
----------------
-"fields": {
-  "my_field": [
-    "v1",
-    "v2"
-  ]
-}
----------------
-
-==== Timezone for date field
-
-Specifying the `time_zone` parameter in queries or aggregations on fields of
-type `date` must now be either an ISO 8601 UTC offset, or a timezone id. For
-example, the value `+1:00` must now be written as `+01:00`.
-
-==== Only highlight queried fields
-
-The default value for the `require_field_match` option has changed from
-`false` to `true`, meaning that the highlighters will, by default, only take
-the fields that were queried into account.
-
-This means that, when querying the `_all` field, trying to highlight on any
-field other than `_all`  will produce no highlighted snippets. Querying the
-same fields that need to be highlighted is the cleaner solution to get
-highlighted snippets back. Otherwise `require_field_match` option can be set
-to `false` to ignore field names completely when highlighting.
-
-The postings highlighter doesn't support the `require_field_match` option
-anymore, it will only highlight fields that were queried.
-
-==== Postings highlighter doesn't support `match_phrase_prefix`
-
-The `match` query with type set to `phrase_prefix` (or the
-`match_phrase_prefix` query) is not supported by the postings highlighter. No
-highlighted snippets will be returned.
-
-
-
diff --git a/docs/reference/migration/migrate_2_0/settings.asciidoc b/docs/reference/migration/migrate_2_0/settings.asciidoc
deleted file mode 100644
index 06aa743a5d8..00000000000
--- a/docs/reference/migration/migrate_2_0/settings.asciidoc
+++ /dev/null
@@ -1,204 +0,0 @@
-[[breaking_20_setting_changes]]
-=== Setting changes
-
-==== Command line flags
-
-Command line flags using single dash notation must be now specified as the first arguments.
-For example if previously using:
-
-[source,sh]
----------------
-./elasticsearch --node.name=test_node -Des.path.conf=/opt/elasticsearch/conf/test_node
----------------
-
-This will now need to be changed to:
-
-[source,sh]
----------------
-./elasticsearch -Des.path.conf=/opt/elasticsearch/conf/test_node --node.name=test_node
----------------
-
-for the flag to take effect.
-
-[[migration-script-settings]]
-==== Scripting settings
-
-The `script.disable_dynamic` node setting has been replaced by fine-grained
-script settings described in the <<enable-dynamic-scripting,scripting docs>>.
-The following setting previously used to enable dynamic or inline scripts:
-
-[source,yaml]
----------------
-script.disable_dynamic: false
----------------
-
-It should be replaced with the following two settings in `elasticsearch.yml` that
-achieve the same result:
-
-[source,yaml]
----------------
-script.inline: true
-script.indexed: true
----------------
-
-==== Units required for time and byte-sized settings
-
-Any settings which accept time or byte values must now be specified with
-units.  For instance, it is too easy to set the `refresh_interval` to 1
-*millisecond* instead of 1 second:
-
-[source,js]
----------------
-PUT _settings
-{
-  "index.refresh_interval": 1
-}
----------------
-
-In 2.0, the above request will throw an exception. Instead the refresh
-interval should be set to `"1s"` for one second.
-
-==== Merge and merge throttling settings
-
-The tiered merge policy is now the only supported merge policy. These settings
-have been removed:
-
-* `index.merge.policy.type`
-* `index.merge.policy.min_merge_size`
-* `index.merge.policy.max_merge_size`
-* `index.merge.policy.merge_factor`
-* `index.merge.policy.max_merge_docs`
-* `index.merge.policy.calibrate_size_by_deletes`
-* `index.merge.policy.min_merge_docs`
-* `index.merge.policy.max_merge_docs`
-
-Merge throttling now uses a feedback loop to auto-throttle.  These settings
-have been removed:
-
-* `indices.store.throttle.type`
-* `indices.store.throttle.max_bytes_per_sec`
-* `index.store.throttle.type`
-* `index.store.throttle.max_bytes_per_sec`
-
-==== Shadow replica settings
-
-The `node.enable_custom_paths` setting has been removed and replaced by the
-`path.shared_data` setting to allow shadow replicas with custom paths to work
-with the security manager. For example, if your previous configuration had:
-
-[source,yaml]
-------
-node.enable_custom_paths: true
-------
-
-And you created an index using shadow replicas with `index.data_path` set to
-`/opt/data/my_index` with the following:
-
-[source,js]
---------------------------------------------------
-PUT /my_index
-{
-  "index": {
-    "number_of_shards": 1,
-    "number_of_replicas": 4,
-    "data_path": "/opt/data/my_index",
-    "shadow_replicas": true
-  }
-}
---------------------------------------------------
-
-For 2.0, you will need to set `path.shared_data` to a parent directory of the
-index's data_path, so:
-
-[source,yaml]
------------
-path.shared_data: /opt/data
------------
-
-==== Resource watcher settings renamed
-
-The setting names for configuring the resource watcher have been renamed
-to prevent clashes with the watcher plugin
-
-* `watcher.enabled` is now `resource.reload.enabled`
-* `watcher.interval` is now `resource.reload.interval`
-* `watcher.interval.low` is now `resource.reload.interval.low`
-* `watcher.interval.medium` is now `resource.reload.interval.medium`
-* `watcher.interval.high` is now `resource.reload.interval.high`
-
-==== index.gateway setting renamed
-
-* `index.gateway.local.sync` is now `index.translog.sync_interval`
-
-==== Hunspell dictionary configuration
-
-The parameter `indices.analysis.hunspell.dictionary.location` has been
-removed, and `<path.conf>/hunspell` is always used.
-
-==== CORS allowed origins
-
-The CORS allowed origins setting, `http.cors.allow-origin`, no longer has a default value. Previously, the default value
-was `*`, which would allow CORS requests from any origin and is considered insecure. The `http.cors.allow-origin` setting
-should be specified with only the origins that should be allowed, like so:
-
-[source,yaml]
----------------
-http.cors.allow-origin: /https?:\/\/localhost(:[0-9]+)?/
----------------
-
-==== JSONP support
-
-JSONP callback support has now been removed. CORS should be used to access Elasticsearch
-over AJAX instead:
-
-[source,yaml]
----------------
-http.cors.enabled: true
-http.cors.allow-origin: /https?:\/\/localhost(:[0-9]+)?/
----------------
-
-==== In memory indices
-
-The `memory` / `ram` store (`index.store.type`) option was removed in
-Elasticsearch.  In-memory indices are no longer supported.
-
-==== Log messages truncated
-
-Log messages are now truncated at 10,000 characters. This can be changed in
-the `logging.yml` configuration file with the `file.layout.conversionPattern`
-setting.
-
-==== Custom config file
-
-It is no longer possible to specify a custom config file with the `CONF_FILE`
-environment variable, or the `-Des.config`, `-Des.default.config`, or
-`-Delasticsearch.config` parameters.
-
-Instead, the config file must be named `elasticsearch.yml` and must be located
-in the default `config/` directory, unless a custom config directory is specified.
-
-The location of a custom config directory may be specified as follows:
-
-[source,sh]
---------------
-./bin/elasticsearch --path.conf=/path/to/conf/dir
-./bin/elasticsearch-plugin -Des.path.conf=/path/to/conf/dir install analysis-icu
---------------
-
-When using the RPM or debian packages, the plugin script and the
-init/service scripts will consult the `CONF_DIR` environment variable
-to check for a custom config location.  The value of the `CONF_DIR`
-variable can be set in the environment config file which is located either in
-`/etc/default/elasticsearch` or `/etc/sysconfig/elasticsearch`.
-
-==== Custom analysis file paths
-
-It is no longer possible to set custom file path outside `CONF_DIR` for `*_path` settings
-in <<analysis-charfilters,char>> or <<analysis-tokenfilters,token>> filters.
-You must specify either relative path to `CONF_DIR` location or absolute path inside `CONF_DIR` location.
-
-==== `ES_CLASSPATH removed`
-
-The `ES_CLASSPATH` environment variable is no longer used to set the class
-path. External libraries should preferably be loaded using the plugin
-mechanism or, if you really must, be copied to the `lib/` directory.
diff --git a/docs/reference/migration/migrate_2_0/snapshot_restore.asciidoc b/docs/reference/migration/migrate_2_0/snapshot_restore.asciidoc
deleted file mode 100644
index c9b222abdc8..00000000000
--- a/docs/reference/migration/migrate_2_0/snapshot_restore.asciidoc
+++ /dev/null
@@ -1,38 +0,0 @@
-[[breaking_20_snapshot_and_restore_changes]]
-=== Snapshot and Restore changes
-
-==== File-system repositories must be whitelisted
-
-Locations of the shared file system repositories and the URL repositories with
-`file:` URLs now have to be registered before starting Elasticsearch using the
-`path.repo` setting. The `path.repo` setting can contain one or more
-repository locations:
-
-[source,yaml]
----------------
-path.repo: ["/mnt/daily", "/mnt/weekly"]
----------------
-
-If the repository location is specified as an absolute path it has to start
-with one of the locations specified in `path.repo`. If the location is
-specified as a relative path, it will be resolved against the first location
-specified in the `path.repo` setting.
-
-==== URL repositories must be whitelisted
-
-URL repositories with `http:`, `https:`, and `ftp:` URLs have to be
-whitelisted before starting Elasticsearch with the
-`repositories.url.allowed_urls` setting. This setting supports wildcards in
-the place of host, path, query, and fragment. For example:
-
-[source,yaml]
------------------------------------
-repositories.url.allowed_urls: ["http://www.example.org/root/*", "https://*.mydomain.com/*?*#*"]
------------------------------------
-
-==== Wildcard expansion
-
-The obsolete parameters `expand_wildcards_open` and `expand_wildcards_close`
-are no longer supported by the snapshot and restore operations. These
-parameters have been replaced by a single `expand_wildcards` parameter. See
-<<multi-index,the multi-index docs>> for more.
diff --git a/docs/reference/migration/migrate_2_0/stats.asciidoc b/docs/reference/migration/migrate_2_0/stats.asciidoc
deleted file mode 100644
index dc80ecd83ec..00000000000
--- a/docs/reference/migration/migrate_2_0/stats.asciidoc
+++ /dev/null
@@ -1,52 +0,0 @@
-[[breaking_20_stats_info_and_literal_cat_literal_changes]]
-=== Stats, info, and `cat` changes
-
-==== Sigar removed
-
-We no longer ship the Sigar library for operating system dependent statistics,
-as it no longer seems to be maintained.  Instead, we rely on the statistics
-provided by the JVM.  This has resulted in a number of changes to the node
-info, and node stats responses:
-
-* `network.*` has been removed from nodes info and nodes stats.
-* `fs.*.dev` and `fs.*.disk*` have been removed from nodes stats.
-* `os.*` has been removed from nodes stats, except for `os.timestamp`,
-  `os.load_average`, `os.mem.*`, and `os.swap.*`.
-* `os.mem.total` and `os.swap.total` have been removed from nodes info.
-* `process.mem.resident` and `process.mem.share` have been removed from node stats.
-
-==== Removed `id_cache` from stats apis
-
-Removed `id_cache` metric from nodes stats, indices stats and cluster stats
-apis. This metric has also been removed from the shards cat, indices cat and
-nodes cat apis. Parent/child memory is now reported under fielddata, because
-it has internally be using fielddata for a while now.
-
-To just see how much parent/child related field data is taking, the
-`fielddata_fields` option can be used on the stats apis. Indices stats
-example:
-
-[source,js]
---------------------------------------------------
-GET /_stats/fielddata?fielddata_fields=_parent
---------------------------------------------------
-
-==== Percolator stats
-
-The total time spent running percolator queries is now called `percolate.time`
-instead of `percolate.get_time`.
-
-==== Cluster state REST API
-
-The cluster state API doesn't return the `routing_nodes` section anymore when
-`routing_table` is requested. The newly introduced `routing_nodes` flag can be
-used separately to control whether `routing_nodes` should be returned.
-
-==== Index status API
-
-The deprecated index status API has been removed.
-
-==== Nodes Stats API
-
-Queue lengths are now reported as basic numeric so they can easily processed by code. Before we used a human
-readable format. For example, a queue with 1,000 items is now reported as `1000` instead of `1k`.
diff --git a/docs/reference/migration/migrate_2_0/striping.asciidoc b/docs/reference/migration/migrate_2_0/striping.asciidoc
deleted file mode 100644
index 2e80f29c774..00000000000
--- a/docs/reference/migration/migrate_2_0/striping.asciidoc
+++ /dev/null
@@ -1,21 +0,0 @@
-[[breaking_20_multiple_literal_data_path_literal_striping]]
-=== Multiple `path.data` striping
-
-Previously, if the `path.data` setting listed multiple data paths, then a
-shard would be ``striped'' across all paths by writing a whole file to each
-path in turn (in accordance with the `index.store.distributor` setting).  The
-result was that files from a single segment in a shard could be spread across
-multiple disks, and the failure of any one disk could corrupt multiple shards.
-
-This striping is no longer supported.  Instead, different shards may be
-allocated to different paths, but all of the files in a single shard will be
-written to the same path.
-
-If striping is detected while starting Elasticsearch 2.0.0 or later, *all of
-the files belonging to the same shard will be migrated to the same path*. If
-there is not enough disk space to complete this migration, the upgrade will be
-cancelled and can only be resumed once enough disk space is made available.
-
-The `index.store.distributor` setting has also been removed.
-
-
diff --git a/docs/reference/migration/migrate_2_1.asciidoc b/docs/reference/migration/migrate_2_1.asciidoc
deleted file mode 100644
index 454a57f96bc..00000000000
--- a/docs/reference/migration/migrate_2_1.asciidoc
+++ /dev/null
@@ -1,87 +0,0 @@
-[[breaking-changes-2.1]]
-== Breaking changes in 2.1
-
-This section discusses the changes that you need to be aware of when migrating
-your application to Elasticsearch 2.1.
-
-* <<breaking_21_search_changes>>
-* <<breaking_21_update_changes>>
-* <<breaking_21_removed_features>>
-* <<breaking_21_more_like_this>>
-* <<breaking_21_nested_sorting>>
-* <<breaking_21_index_apis>>
-
-[[breaking_21_search_changes]]
-=== Search changes
-
-==== `search_type=scan` deprecated
-
-The `scan` search type has been deprecated. All benefits from this search
-type can now be achieved by doing a scroll request that sorts documents in
-`_doc` order, for instance:
-
-[source,sh]
----------------
-GET /my_index/_search?scroll=2m
-{
-  "sort": [
-    "_doc"
-  ]
-}
----------------
-
-Scroll requests sorted by `_doc` have been optimized to more efficiently resume
-from where the previous request stopped, so this will have the same performance
-characteristics as the former `scan` search type.
-
-==== from + size limits
-
-Elasticsearch will now return an error message if a query's `from` + `size` is
-more than the `index.max_result_window` parameter. This parameter defaults to
-10,000 which is safe for almost all clusters. Values higher than can consume
-significant chunks of heap memory per search and per shard executing the
-search. It's safest to leave this value as it is an use the scroll api for any
-deep scrolling but this setting is dynamic so it can raised or lowered as
-needed.
-
-[[breaking_21_update_changes]]
-=== Update changes
-
-==== Updates now `detect_noop` by default
-
-We've switched the default value of the `detect_noop` option from `false` to
-`true`. This means that Elasticsearch will ignore updates that don't change
-source unless you explicitly set `"detect_noop": false`. `detect_noop` was
-always computationally cheap compared to the expense of the update which can be
-thought of as a delete operation followed by an index operation.
-
-[[breaking_21_removed_features]]
-=== Removed features
-
-==== `indices.fielddata.cache.expire`
-
-The experimental feature `indices.fielddata.cache.expire` has been removed.
-For indices that have this setting configured, this config will be ignored.
-
-[[breaking_21_more_like_this]]
-=== More Like This
-
-The MoreLikeThisQueryBuilder#ignoreLike methods have been deprecated in favor
-of using the unlike methods.
-
-MoreLikeThisBuilder#addItem has been deprecated in favor of using
-MoreLikeThisBuilder#addLikeItem.
-
-[[breaking_21_nested_sorting]]
-=== Nested sorting
-
-If sorting on field inside a nested object then the `nested_path` should be specified.
-Before there was an attempt to resolve the nested path automatically, but that was sometimes incorrect.
-To avoid confusion the `nested_path` should always be specified.
-
-[[breaking_21_index_apis]]
-=== Index APIs
-
-==== Optimize API
-
-The Optimize API has been deprecated, all new optimize actions should use the new Force Merge API.
diff --git a/docs/reference/migration/migrate_2_2.asciidoc b/docs/reference/migration/migrate_2_2.asciidoc
deleted file mode 100644
index 39c059e7f47..00000000000
--- a/docs/reference/migration/migrate_2_2.asciidoc
+++ /dev/null
@@ -1,55 +0,0 @@
-[[breaking-changes-2.2]]
-== Breaking changes in 2.2
-
-This section discusses the changes that you need to be aware of when migrating
-your application to Elasticsearch 2.2.
-
-[float]
-=== Scripting and security
-
-The Java Security Manager is being used to lock down the privileges available
-to the scripting languages and to restrict the classes they are allowed to
-load to a predefined whitelist.  These changes may cause scripts which worked
-in earlier versions to fail.  See <<modules-scripting-security>> for more
-details.
-
-[float]
-=== Field stats API
-
-The field stats' response format has been changed for number based and date
-fields. The `min_value` and `max_value` elements now return values as number
-and the new `min_value_as_string` and `max_value_as_string` return the values
-as string.
-
-[float]
-=== Default logging using systemd
-
-In previous versions of Elasticsearch using systemd, the default logging
-configuration routed standard output to `/dev/null` and standard error to
-the journal. However, there are often critical error messages at
-startup that are logged to standard output rather than standard error
-and these error messages would be lost to the ether. The default has
-changed to now route standard output to the journal and standard error
-to inherit this setting (these are the defaults for systemd). These
-settings can be modified by editing the `elasticsearch.service` file.
-
-[float]
-=== Cloud AWS Plugin
-
-Proxy settings have been deprecated and renamed:
-
-* from `cloud.aws.proxy_host` to `cloud.aws.proxy.host`
-* from `cloud.aws.ec2.proxy_host` to `cloud.aws.ec2.proxy.host`
-* from `cloud.aws.s3.proxy_host` to `cloud.aws.s3.proxy.host`
-* from `cloud.aws.proxy_port` to `cloud.aws.proxy.port`
-* from `cloud.aws.ec2.proxy_port` to `cloud.aws.ec2.proxy.port`
-* from `cloud.aws.s3.proxy_port` to `cloud.aws.s3.proxy.port`
-
-If you are using proxy settings, update your settings as deprecated ones will
-be removed in next major version.
-
-[float]
-=== Multicast plugin deprecated
-
-The `discovery-multicast` plugin has been deprecated in 2.2.0 and has
-been removed in 3.0.0.
diff --git a/docs/reference/migration/migrate_2_3.asciidoc b/docs/reference/migration/migrate_2_3.asciidoc
deleted file mode 100644
index 0d741e2adb2..00000000000
--- a/docs/reference/migration/migrate_2_3.asciidoc
+++ /dev/null
@@ -1,19 +0,0 @@
-[[breaking-changes-2.3]]
-== Breaking changes in 2.3
-
-This section discusses the changes that you need to be aware of when migrating
-your application to Elasticsearch 2.3.
-
-* <<breaking_23_index_apis>>
-
-[[breaking_23_index_apis]]
-=== Mappings
-
-==== Limit to the number of `nested` fields
-
-Indexing a document with 100 nested fields actually indexes 101 documents as each nested
-document is indexed as a separate document. To safeguard against ill-defined mappings
-the number of nested fields that can be defined per index has been limited to 50.
-This default limit can be changed with the index setting `index.mapping.nested_fields.limit`.
-Note that the limit is only checked when new indices are created or mappings are updated. It
-will thus only affect existing pre-2.3 indices if their mapping is changed.
diff --git a/docs/reference/migration/migrate_5_0.asciidoc b/docs/reference/migration/migrate_5_0.asciidoc
index 6fcf566fdcb..f4aaacee35e 100644
--- a/docs/reference/migration/migrate_5_0.asciidoc
+++ b/docs/reference/migration/migrate_5_0.asciidoc
@@ -4,843 +4,63 @@
 This section discusses the changes that you need to be aware of when migrating
 your application to Elasticsearch 5.0.
 
+[float]
+=== Indices created before 5.0
+
+Elasticsearch 5.0 can read indices created in version 2.0 and above.  If any
+of your indices were created before 2.0 you will need to upgrade to the
+latest 2.x version of Elasticsearch first, in order to upgrade your indices or
+to delete the old indices. Elasticsearch will not start in the presence of old
+indices. To upgrade 2.x indices, first start a node which have access to all
+the data folders and let it upgrade all the indices before starting up rest of
+the cluster.
+
+[IMPORTANT]
+.Reindex indices from Elasticseach 1.x or before
+=========================================
+
+Indices created in Elasticsearch 1.x or before will need to be reindexed with
+Elasticsearch 2.x in order to be readable by Elasticsearch 5.x. The easiest
+way to do this is to upgrade to Elasticsearch 2.3 or later and to use the
+`reindex` API.
+
+=========================================
+
+[float]
+=== Also see:
+
 * <<breaking_50_search_changes>>
+* <<breaking_50_mapping_changes>>
+* <<breaking_50_percolator>>
+* <<breaking_50_index_apis>>
+* <<breaking_50_settings_changes>>
+* <<breaking_50_allocation>>
 * <<breaking_50_rest_api_changes>>
 * <<breaking_50_cat_api>>
-* <<breaking_50_parent_child_changes>>
-* <<breaking_50_settings_changes>>
-* <<breaking_50_mapping_changes>>
-* <<breaking_50_plugins>>
 * <<breaking_50_java_api_changes>>
-* <<breaking_50_cache_concurrency>>
-* <<breaking_50_non_loopback>>
-* <<breaking_50_thread_pool>>
-* <<breaking_50_allocation>>
-* <<breaking_50_percolator>>
 * <<breaking_50_packaging>>
-* <<breaking_50_scripting>>
-* <<breaking_50_term_vectors>>
-* <<breaking_50_security>>
+* <<breaking_50_plugins>>
 
-[[breaking_50_search_changes]]
-=== Warmers
+include::migrate_5_0/search.asciidoc[]
 
-Thanks to several changes like doc values by default or disk-based norms,
-warmers have become quite useless. As a consequence, warmers and the warmer
-API have been removed: it is not possible anymore to register queries that
-will run before a new IndexSearcher is published.
+include::migrate_5_0/mapping.asciidoc[]
 
-Don't worry if you have warmers defined on your indices, they will simply be
-ignored when upgrading to 5.0.
+include::migrate_5_0/percolator.asciidoc[]
 
-=== Search changes
+include::migrate_5_0/index-apis.asciidoc[]
 
-==== `search_type=count` removed
+include::migrate_5_0/settings.asciidoc[]
 
-The `count` search type was deprecated since version 2.0.0 and is now removed.
-In order to get the same benefits, you just need to set the value of the `size`
-parameter to `0`.
+include::migrate_5_0/allocation.asciidoc[]
 
-For instance, the following request:
+include::migrate_5_0/rest.asciidoc[]
 
-[source,sh]
----------------
-GET /my_index/_search?search_type=count
-{
-  "aggs": {
-    "my_terms": {
-       "terms": {
-         "field": "foo"
-       }
-     }
-  }
-}
----------------
+include::migrate_5_0/cat.asciidoc[]
 
-can be replaced with:
+include::migrate_5_0/java.asciidoc[]
 
-[source,sh]
----------------
-GET /my_index/_search
-{
-  "size": 0,
-  "aggs": {
-    "my_terms": {
-       "terms": {
-         "field": "foo"
-       }
-     }
-  }
-}
----------------
+include::migrate_5_0/packaging.asciidoc[]
 
-==== `search_type=scan` removed
+include::migrate_5_0/plugins.asciidoc[]
 
-The `scan` search type was deprecated since version 2.1.0 and is now removed.
-All benefits from this search type can now be achieved by doing a scroll
-request that sorts documents in `_doc` order, for instance:
 
-[source,sh]
----------------
-GET /my_index/_search?scroll=2m
-{
-  "sort": [
-    "_doc"
-  ]
-}
----------------
-
-Scroll requests sorted by `_doc` have been optimized to more efficiently resume
-from where the previous request stopped, so this will have the same performance
-characteristics as the former `scan` search type.
-
-==== Boost accuracy for queries on `_all`
-
-Per-field boosts on the `_all` are now compressed on a single byte instead of
-4 bytes previously. While this will make the index more space-efficient, this
-also means that the boosts will be less accurately encoded.
-
-[[breaking_50_rest_api_changes]]
-=== REST API changes
-
-==== id values longer than 512 bytes are rejected
-
-When specifying an `_id` value longer than 512 bytes, the request will be
-rejected.
-
-==== search exists api removed
-
-The search exists api has been removed in favour of using the search api with
-`size` set to `0` and `terminate_after` set to `1`.
-
-==== `/_optimize` endpoint removed
-
-The deprecated `/_optimize` endpoint has been removed. The `/_forcemerge`
-endpoint should be used in lieu of optimize.
-
-The `GET` HTTP verb for `/_forcemerge` is no longer supported, please use the
-`POST` HTTP verb.
-
-==== Deprecated queries removed
-
-The following deprecated queries have been removed:
-
-* `filtered`: use `bool` query instead, which supports `filter` clauses too
-* `and`: use `must` clauses in a `bool` query instead
-* `or`: use should clauses in a `bool` query instead
-* `limit`: use `terminate_after` parameter instead
-* `fquery`: obsolete after filters and queries have been merged
-* `query`: obsolete after filters and queries have been merged
-
-==== Unified fuzziness parameter
-
-* Removed support for the deprecated `min_similarity` parameter in `fuzzy query`, in favour of `similarity`.
-* Removed support for the deprecated `fuzzy_min_sim` parameter in `query_string` query, in favour of `similarity`.
-* Removed support for the deprecated `edit_distance` parameter in completion suggester, in favour of `similarity`.
-
-==== indices query
-
-Removed support for the deprecated `filter` and `no_match_filter` fields in `indices` query,
-in favour of `query` and `no_match_query`.
-
-==== nested query
-
-Removed support for the deprecated `filter` fields in `nested` query, in favour of `query`.
-
-==== terms query
-
-Removed support for the deprecated `minimum_should_match` and `disable_coord` in `terms` query, use `bool` query instead.
-Removed also support for the deprecated `execution` parameter.
-
-==== function_score query
-
-Removed support for the top level `filter` element in `function_score` query, replaced by `query`.
-
-==== highlighters
-
-Removed support for multiple highlighter names, the only supported ones are: `plain`, `fvh` and `postings`.
-
-==== top level filter
-
-Removed support for the deprecated top level `filter` in the search api, replaced by `post_filter`.
-
-==== `query_binary` and `filter_binary` removed
-
-Removed support for the undocumented `query_binary` and `filter_binary` sections of a search request.
-
-==== `span_near`'s' `collect_payloads` deprecated
-
-Payloads are now loaded when needed.
-
-[[breaking_50_cat_api]]
-=== CAT API changes
-
-==== Use Accept header for specifying response media type
-
-Previous versions of Elasticsearch accepted the Content-type header
-field for controlling the media type of the response in the cat API.
-This is in opposition to the HTTP spec which specifies the Accept
-header field for this purpose. Elasticsearch now uses the Accept header
-field and support for using the Content-Type header field for this
-purpose has been removed.
-
-==== Host field removed from the cat nodes API
-
-The `host` field has been removed from the cat nodes API as its value
-is always equal to the `ip` field. The `name` field is available in the
-cat nodes API and should be used instead of the `host` field.
-
-[[breaking_50_parent_child_changes]]
-=== Parent/Child changes
-
-The `children` aggregation, parent child inner hits and `has_child` and `has_parent` queries will not work on indices
-with `_parent` field mapping created before version `2.0.0`. The data of these indices need to be re-indexed into a new index.
-
-The format of the join between parent and child documents have changed with the `2.0.0` release. The old
-format can't read from version `5.0.0` and onwards. The new format allows for a much more efficient and
-scalable join between parent and child documents and the join data structures are stored on disk
-data structures as opposed as before the join data structures were stored in the jvm heap space.
-
-==== `score_type` has been removed
-
-The `score_type` option has been removed from the `has_child` and `has_parent` queries in favour of the `score_mode` option
-which does the exact same thing.
-
-==== `sum` score mode removed
-
-The `sum` score mode has been removed in favour of the `total` mode which does the same and is already available in
-previous versions.
-
-==== `max_children` option
-
-When `max_children` was set to `0` on the `has_child` query then there was no upper limit on how many children documents
-are allowed to match. This has changed and `0` now really means to zero child documents are allowed. If no upper limit
-is needed then the `max_children` option shouldn't be defined at all on the `has_child` query.
-
-==== `_parent` field no longer indexed
-
-The join between parent and child documents no longer relies on indexed fields and therefor from `5.0.0` onwards
-the `_parent` indexed field won't be indexed. In order to find documents that referrer to a specific parent id
-the new `parent_id` query can be used. The get response and hits inside the search response remain to include
-the parent id under the `_parent` key.
-
-[[breaking_50_settings_changes]]
-=== Settings changes
-
-From Elasticsearch 5.0 on all settings are validated before they are applied. Node level and default index
-level settings are validated on node startup, dynamic cluster and index setting are validated before they are updated/added
-to the cluster state. Every setting must be a _known_ setting or in other words all settings must be registered with the
-node or transport client they are used with. This implies that plugins that define custom settings must register all of their
-settings during pluging loading using the `SettingsModule#registerSettings(Setting)` method.
-
-==== Node settings
-
-The `name` setting has been removed and is replaced by `node.name`. Usage of `-Dname=some_node_name` is not supported
-anymore.
-
-==== Transport Settings
-
-All settings with a `netty` infix have been replaced by their already existing `transport` synonyms. For instance `transport.netty.bind_host` is
-no longer supported and should be replaced by the superseding setting `transport.bind_host`.
-
-==== Analysis settings
-
-The `index.analysis.analyzer.default_index` analyzer is not supported anymore.
-If you wish to change the analyzer to use for indexing, change the
-`index.analysis.analyzer.default` analyzer instead.
-
-==== Ping timeout settings
-
-Previously, there were three settings for the ping timeout: `discovery.zen.initial_ping_timeout`,
-`discovery.zen.ping.timeout` and `discovery.zen.ping_timeout`. The former two have been removed and
-the only setting key for the ping timeout is now `discovery.zen.ping_timeout`. The default value for
-ping timeouts remains at three seconds.
-
-==== Recovery settings
-
-Recovery settings deprecated in 1.x have been removed:
-
- * `index.shard.recovery.translog_size` is superseded by `indices.recovery.translog_size`
- * `index.shard.recovery.translog_ops` is superseded by `indices.recovery.translog_ops`
- * `index.shard.recovery.file_chunk_size` is superseded by `indices.recovery.file_chunk_size`
- * `index.shard.recovery.concurrent_streams` is superseded by `indices.recovery.concurrent_streams`
- * `index.shard.recovery.concurrent_small_file_streams` is superseded by `indices.recovery.concurrent_small_file_streams`
- * `indices.recovery.max_size_per_sec` is superseded by `indices.recovery.max_bytes_per_sec`
-
-If you are using any of these settings please take the time and review their purpose. All of the settings above are considered
-_expert settings_ and should only be used if absolutely necessary. If you have set any of the above setting as persistent
-cluster settings please use the settings update API and set their superseded keys accordingly.
-
-The following settings have been removed without replacement
-
- * `indices.recovery.concurrent_small_file_streams` - recoveries are now single threaded. The number of concurrent outgoing recoveries are throttled via allocation deciders
- * `indices.recovery.concurrent_file_streams` - recoveries are now single threaded. The number of concurrent outgoing recoveries are throttled via allocation deciders
-
-==== Translog settings
-
-The `index.translog.flush_threshold_ops` setting is not supported anymore. In order to control flushes based on the transaction log
-growth use `index.translog.flush_threshold_size` instead. Changing the translog type with `index.translog.fs.type` is not supported
-anymore, the `buffered` implementation is now the only available option and uses a fixed `8kb` buffer.
-
-The translog by default is fsynced on a request basis such that the ability to fsync on every operation is not necessary anymore. In-fact it can
-be a performance bottleneck and it's trappy since it enabled by a special value set on `index.translog.sync_interval`. `index.translog.sync_interval`
-now doesn't accept a value less than `100ms` which prevents fsyncing too often if async durability is enabled. The special value `0` is not supported anymore.
-
-==== Request Cache Settings
-
-The deprecated settings `index.cache.query.enable` and `indices.cache.query.size` have been removed and are replaced with
-`index.requests.cache.enable` and `indices.requests.cache.size` respectively.
-
-`indices.requests.cache.clean_interval` has been replaced with `indices.cache.clean_interval` and is no longer supported.
-
-==== Field Data Cache Settings
-
-`indices.fielddata.cache.clean_interval` has been replaced with `indices.cache.clean_interval` and is no longer supported.
-
-==== Allocation settings
-
-Allocation settings deprecated in 1.x have been removed:
-
- * `cluster.routing.allocation.concurrent_recoveries` is superseded by `cluster.routing.allocation.node_concurrent_recoveries`
-
-Please change the setting in your configuration files or in the clusterstate to use the new settings instead.
-
-==== Similarity settings
-
-The 'default' similarity has been renamed to 'classic'.
-
-==== Indexing settings
-
-`indices.memory.min_shard_index_buffer_size` and `indices.memory.max_shard_index_buffer_size` are removed since Elasticsearch now allows any one shard to any
-amount of heap as long as the total indexing buffer heap used across all shards is below the node's `indices.memory.index_buffer_size` (default: 10% of the JVM heap)
-
-==== Removed es.max-open-files
-
-Setting the system property es.max-open-files to true to get
-Elasticsearch to print the number of maximum open files for the
-Elasticsearch process has been removed. This same information can be
-obtained from the <<cluster-nodes-info>> API, and a warning is logged
-on startup if it is set too low.
-
-==== Removed es.netty.gathering
-
-Disabling Netty from using NIO gathering could be done via the escape
-hatch of setting the system property "es.netty.gathering" to "false".
-Time has proven enabling gathering by default is a non-issue and this
-non-documented setting has been removed.
-
-==== Removed es.useLinkedTransferQueue
-
-The system property `es.useLinkedTransferQueue` could be used to
-control the queue implementation used in the cluster service and the
-handling of ping responses during discovery. This was an undocumented
-setting and has been removed.
-
-[[breaking_50_mapping_changes]]
-=== Mapping changes
-
-==== Default doc values settings
-
-Doc values are now also on by default on numeric and boolean fields that are
-not indexed.
-
-==== Transform removed
-
-The `transform` feature from mappings has been removed. It made issues very hard to debug.
-
-==== Default number mappings
-
-When a floating-point number is encountered, it is now dynamically mapped as a
-float by default instead of a double. The reasoning is that floats should be
-more than enough for most cases but would decrease storage requirements
-significantly.
-
-==== `index` property
-
-On all types but `string`, the `index` property now only accepts `true`/`false`
-instead of `not_analyzed`/`no`. The `string` field still accepts
-`analyzed`/`not_analyzed`/`no`.
-
-==== ++_source++'s `format` option
-
-The `_source` mapping does not support the `format` option anymore. This option
-will still be accepted for indices created before the upgrade to 5.0 for backward
-compatibility, but it will have no effect. Indices created on or after 5.0 will
-reject this option.
-
-==== Object notation
-
-Core types don't support the object notation anymore, which allowed to provide
-values as follows:
-
-[source,json]
----------------
-{
-  "value": "field_value",
-  "boost": 42
-}
----------------
-
-==== `fielddata.format`
-
-Setting `fielddata.format: doc_values` in the mappings used to implicitly
-enable doc values on a field. This no longer works: the only way to enable or
-disable doc values is by using the `doc_values` property of mappings.
-
-
-[[breaking_50_plugins]]
-=== Plugin changes
-
-The command `bin/plugin` has been renamed to `bin/elasticsearch-plugin`.
-The structure of the plugin has changed. All the plugin files must be contained in a directory called `elasticsearch`.
-If you use the gradle build, this structure is automatically generated.
-
-==== Site plugins removed
-
-Site plugins have been removed. It is recommended to migrate site plugins to Kibana plugins.
-
-==== Multicast plugin removed
-
-Multicast has been removed. Use unicast discovery, or one of the cloud discovery plugins.
-
-==== Plugins with custom query implementations
-
-Plugins implementing custom queries need to implement the `fromXContent(QueryParseContext)` method in their
-`QueryParser` subclass rather than `parse`. This method will take care of parsing the query from `XContent` format
-into an intermediate query representation that can be streamed between the nodes in binary format, effectively the
-query object used in the java api. Also, the query parser needs to implement the `getBuilderPrototype` method that
-returns a prototype of the `NamedWriteable` query, which allows to deserialize an incoming query by calling
-`readFrom(StreamInput)` against it, which will create a new object, see usages of `Writeable`. The `QueryParser`
-also needs to declare the generic type of the query that it supports and it's able to parse.
-The query object can then transform itself into a lucene query through the new `toQuery(QueryShardContext)` method,
-which returns a lucene query to be executed on the data node.
-
-Similarly, plugins implementing custom score functions need to implement the `fromXContent(QueryParseContext)`
-method in their `ScoreFunctionParser` subclass rather than `parse`. This method will take care of parsing
-the function from `XContent` format into an intermediate function representation that can be streamed between
-the nodes in binary format, effectively the function object used in the java api. Also, the query parser needs
-to implement the `getBuilderPrototype` method that returns a prototype of the `NamedWriteable` function, which
-allows to deserialize an incoming function by calling `readFrom(StreamInput)` against it, which will create a
-new object, see usages of `Writeable`. The `ScoreFunctionParser` also needs to declare the generic type of the
-function that it supports and it's able to parse. The function object can then transform itself into a lucene
-function through the new `toFunction(QueryShardContext)` method, which returns a lucene function to be executed
-on the data node.
-
-==== Cloud AWS plugin changes
-
-Cloud AWS plugin has been split in two plugins:
-
-* {plugins}/discovery-ec2.html[Discovery EC2 plugin]
-* {plugins}/repository-s3.html[Repository S3 plugin]
-
-Proxy settings for both plugins have been renamed:
-
-* from `cloud.aws.proxy_host` to `cloud.aws.proxy.host`
-* from `cloud.aws.ec2.proxy_host` to `cloud.aws.ec2.proxy.host`
-* from `cloud.aws.s3.proxy_host` to `cloud.aws.s3.proxy.host`
-* from `cloud.aws.proxy_port` to `cloud.aws.proxy.port`
-* from `cloud.aws.ec2.proxy_port` to `cloud.aws.ec2.proxy.port`
-* from `cloud.aws.s3.proxy_port` to `cloud.aws.s3.proxy.port`
-
-==== Cloud Azure plugin changes
-
-Cloud Azure plugin has been split in three plugins:
-
-* {plugins}/discovery-azure.html[Discovery Azure plugin]
-* {plugins}/repository-azure.html[Repository Azure plugin]
-* {plugins}/store-smb.html[Store SMB plugin]
-
-If you were using the `cloud-azure` plugin for snapshot and restore, you had in `elasticsearch.yml`:
-
-[source,yaml]
------
-cloud:
-    azure:
-        storage:
-            account: your_azure_storage_account
-            key: your_azure_storage_key
------
-
-You need to give a unique id to the storage details now as you can define multiple storage accounts:
-
-[source,yaml]
------
-cloud:
-    azure:
-        storage:
-            my_account:
-                account: your_azure_storage_account
-                key: your_azure_storage_key
------
-
-
-==== Cloud GCE plugin changes
-
-Cloud GCE plugin has been renamed to {plugins}/discovery-gce.html[Discovery GCE plugin].
-
-
-==== Mapper Attachments plugin deprecated
-
-Mapper attachments has been deprecated. Users should use now the {plugins}/ingest-attachment.html[`ingest-attachment`]
-plugin.
-
-
-[[breaking_50_java_api_changes]]
-=== Java API changes
-
-==== Count api has been removed
-
-The deprecated count api has been removed from the Java api, use the search api instead and set size to 0.
-
-The following call
-
-[source,java]
------
-client.prepareCount(indices).setQuery(query).get();
------
-
-can be replaced with
-
-[source,java]
------
-client.prepareSearch(indices).setSource(new SearchSourceBuilder().size(0).query(query)).get();
------
-
-==== BoostingQueryBuilder
-
-Removed setters for mandatory positive/negative query. Both arguments now have
-to be supplied at construction time already and have to be non-null.
-
-==== SpanContainingQueryBuilder
-
-Removed setters for mandatory big/little inner span queries. Both arguments now have
-to be supplied at construction time already and have to be non-null. Updated
-static factory methods in QueryBuilders accordingly.
-
-==== SpanOrQueryBuilder
-
-Making sure that query contains at least one clause by making initial clause mandatory
-in constructor.
-
-==== SpanNearQueryBuilder
-
-Removed setter for mandatory slop parameter, needs to be set in constructor now. Also
-making sure that query contains at least one clause by making initial clause mandatory
-in constructor. Updated the static factory methods in QueryBuilders accordingly.
-
-==== SpanNotQueryBuilder
-
-Removed setter for mandatory include/exclude span query clause, needs to be set in constructor now.
-Updated the static factory methods in QueryBuilders and tests accordingly.
-
-==== SpanWithinQueryBuilder
-
-Removed setters for mandatory big/little inner span queries. Both arguments now have
-to be supplied at construction time already and have to be non-null. Updated
-static factory methods in QueryBuilders accordingly.
-
-==== QueryFilterBuilder
-
-Removed the setter `queryName(String queryName)` since this field is not supported
-in this type of query. Use `FQueryFilterBuilder.queryName(String queryName)` instead
-when in need to wrap a named query as a filter.
-
-==== WrapperQueryBuilder
-
-Removed `wrapperQueryBuilder(byte[] source, int offset, int length)`. Instead simply
-use  `wrapperQueryBuilder(byte[] source)`. Updated the static factory methods in
-QueryBuilders accordingly.
-
-==== QueryStringQueryBuilder
-
-Removed ability to pass in boost value using `field(String field)` method in form e.g. `field^2`.
-Use the `field(String, float)` method instead.
-
-==== Operator
-
-Removed the enums called `Operator` from `MatchQueryBuilder`, `QueryStringQueryBuilder`,
-`SimpleQueryStringBuilder`, and `CommonTermsQueryBuilder` in favour of using the enum
-defined in `org.elasticsearch.index.query.Operator` in an effort to consolidate the
-codebase and avoid duplication.
-
-==== queryName and boost support
-
-Support for `queryName` and `boost` has been streamlined to all of the queries. That is
-a breaking change till queries get sent over the network as serialized json rather
-than in `Streamable` format. In fact whenever additional fields are added to the json
-representation of the query, older nodes might throw error when they find unknown fields.
-
-==== InnerHitsBuilder
-
-InnerHitsBuilder now has a dedicated addParentChildInnerHits and addNestedInnerHits methods
-to differentiate between inner hits for nested vs. parent / child documents. This change
-makes the type / path parameter mandatory.
-
-==== MatchQueryBuilder
-
-Moving MatchQueryBuilder.Type and MatchQueryBuilder.ZeroTermsQuery enum to MatchQuery.Type.
-Also reusing new Operator enum.
-
-==== MoreLikeThisQueryBuilder
-
-Removed `MoreLikeThisQueryBuilder.Item#id(String id)`, `Item#doc(BytesReference doc)`,
-`Item#doc(XContentBuilder doc)`. Use provided constructors instead.
-
-Removed `MoreLikeThisQueryBuilder#addLike` in favor of texts and/or items being provided
-at construction time. Using arrays there instead of lists now.
-
-Removed `MoreLikeThisQueryBuilder#addUnlike` in favor to using the `unlike` methods
-which take arrays as arguments now rather than the lists used before.
-
-The deprecated `docs(Item... docs)`, `ignoreLike(Item... docs)`,
-`ignoreLike(String... likeText)`, `addItem(Item... likeItems)` have been removed.
-
-==== GeoDistanceQueryBuilder
-
-Removing individual setters for lon() and lat() values, both values should be set together
- using point(lon, lat).
-
-==== GeoDistanceRangeQueryBuilder
-
-Removing setters for to(Object ...) and from(Object ...) in favour of the only two allowed input
-arguments (String, Number). Removing setter for center point (point(), geohash()) because parameter
-is mandatory and should already be set in constructor.
-Also removing setters for lt(), lte(), gt(), gte() since they can all be replaced by equivalent
-calls to to/from() and inludeLower()/includeUpper().
-
-==== GeoPolygonQueryBuilder
-
-Require shell of polygon already to be specified in constructor instead of adding it pointwise.
-This enables validation, but makes it necessary to remove the addPoint() methods.
-
-==== MultiMatchQueryBuilder
-
-Moving MultiMatchQueryBuilder.ZeroTermsQuery enum to MatchQuery.ZeroTermsQuery.
-Also reusing new Operator enum.
-
-Removed ability to pass in boost value using `field(String field)` method in form e.g. `field^2`.
-Use the `field(String, float)` method instead.
-
-==== MissingQueryBuilder
-
-The MissingQueryBuilder which was deprecated in 2.2.0 is removed. As a replacement use ExistsQueryBuilder
-inside a mustNot() clause. So instead of using `new ExistsQueryBuilder(name)` now use
-`new BoolQueryBuilder().mustNot(new ExistsQueryBuilder(name))`.
-
-==== NotQueryBuilder
-
-The NotQueryBuilder which was deprecated in 2.1.0 is removed. As a replacement use BoolQueryBuilder
-with added mustNot() clause. So instead of using `new NotQueryBuilder(filter)` now use
-`new BoolQueryBuilder().mustNot(filter)`.
-
-==== TermsQueryBuilder
-
-Remove the setter for `termsLookup()`, making it only possible to either use a TermsLookup object or
-individual values at construction time. Also moving individual settings for the TermsLookup (lookupIndex,
-lookupType, lookupId, lookupPath) to the separate TermsLookup class, using constructor only and moving
-checks for validation there. Removed `TermsLookupQueryBuilder` in favour of `TermsQueryBuilder`.
-
-==== FunctionScoreQueryBuilder
-
-`add` methods have been removed, all filters and functions must be provided as constructor arguments by
-creating an array of `FunctionScoreQueryBuilder.FilterFunctionBuilder` objects, containing one element
-for each filter/function pair.
-
-`scoreMode` and `boostMode` can only be provided using corresponding enum members instead
-of string values: see `FilterFunctionScoreQuery.ScoreMode` and `CombineFunction`.
-
-`CombineFunction.MULT` has been renamed to `MULTIPLY`.
-
-==== IdsQueryBuilder
-
-For simplicity, only one way of adding the ids to the existing list (empty by default)  is left: `addIds(String...)`
-
-==== DocumentAlreadyExistsException removed
-
-`DocumentAlreadyExistsException` is removed and a `VersionConflictException` is thrown instead (with a better
-error description). This will influence code that use the `IndexRequest.opType()` or `IndexRequest.create()`
-to index a document only if it doesn't already exist.
-
-==== ShapeBuilders
-
-`InternalLineStringBuilder` is removed in favour of `LineStringBuilder`, `InternalPolygonBuilder` in favour of PolygonBuilder` and `Ring` has been replaced with `LineStringBuilder`. Also the abstract base classes `BaseLineStringBuilder` and `BasePolygonBuilder` haven been merged with their corresponding implementations.
-
-==== RescoreBuilder
-
-`RecoreBuilder.Rescorer` was merged with `RescoreBuilder`, which now is an abstract superclass. QueryRescoreBuilder currently is its only implementation.
-
-==== PhraseSuggestionBuilder
-
-The inner DirectCandidateGenerator class has been moved out to its own class called DirectCandidateGeneratorBuilder.
-
-==== Elasticsearch will no longer detect logging implementations
-
-Elasticsearch now logs only to log4j 1.2. Previously if log4j wasn't on the classpath it made some effort to degrade to
-slf4j or java.util.logging. Now it'll fail to work without the log4j 1.2 api. The log4j-over-slf4j bridge ought to work
-when using the java client. As should log4j 2's log4j-1.2-api. The Elasticsearch server now only supports log4j as
-configured by logging.yml and it no longer makes any effort to work if log4j isn't present.
-
-[[breaking_50_cache_concurrency]]
-=== Cache concurrency level settings removed
-
-Two cache concurrency level settings `indices.requests.cache.concurrency_level` and
-`indices.fielddata.cache.concurrency_level` because they no longer apply to the cache implementation used for the
-request cache and the field data cache.
-
-[[breaking_50_non_loopback]]
-=== Remove bind option of `non_loopback`
-
-This setting would arbitrarily pick the first interface not marked as loopback. Instead, specify by address
-scope (e.g. `_local_,_site_` for all loopback and private network addresses) or by explicit interface names,
-hostnames, or addresses.
-
-[[breaking_50_thread_pool]]
-=== Forbid changing of thread pool types
-
-Previously, <<modules-threadpool,thread pool types>> could be dynamically adjusted. The thread pool type effectively
-controls the backing queue for the thread pool and modifying this is an expert setting with minimal practical benefits
-and high risk of being misused. The ability to change the thread pool type for any thread pool has been removed; do note
-that it is still possible to adjust relevant thread pool parameters for each of the thread pools (e.g., depending on
-the thread pool type, `keep_alive`, `queue_size`, etc.).
-
-[[breaking_50_cpu_stats]]
-=== System CPU stats
-
-The recent CPU usage (as a percent) has been added to the OS stats
-reported under the node stats API and the cat nodes API. The breaking
-change here is that there is a new object in the `os` object in the node
-stats response. This object is called `cpu` and includes "percent" and
-`load_average` as fields. This moves the `load_average` field that was
-previously a top-level field in the `os` object to the `cpu` object. The
-format of the `load_average` field has changed to an object with fields
-`1m`, `5m`, and `15m` representing the one-minute, five-minute and
-fifteen-minute loads respectively. If any of these fields are not present,
-it indicates that the corresponding value is not available.
-
-In the cat nodes API response, the `cpu` field is output by default. The
-previous `load` field has been removed and is replaced by `load_1m`,
-`load_5m`, and `load_15m` which represent the one-minute, five-minute
-and fifteen-minute loads respectively. The field will be null if the
-corresponding value is not available.
-
-Finally, the API for `org.elasticsearch.monitor.os.OsStats` has
-changed. The `getLoadAverage` method has been removed. The value for
-this can now be obtained from `OsStats.Cpu#getLoadAverage` but it is no
-longer a double and is instead an object encapsulating the one-minute,
-five-minute and fifteen-minute load averages. Additionally, the recent
-CPU usage can be obtained from `OsStats.Cpu#getPercent`.
-
-=== Fields option
-Only stored fields are retrievable with this option.
-The fields option won't be able to load non stored fields from _source anymore.
-
-[[breaking_50_allocation]]
-=== Primary shard allocation
-
-Previously, primary shards were only assigned if a quorum of shard copies were found (configurable using
-`index.recovery.initial_shards`, now deprecated). In case where a primary had only a single replica, quorum was defined
-to be a single shard. This meant that any shard copy of an index with replication factor 1 could become primary, even it
-was a stale copy of the data on disk. This is now fixed by using allocation IDs.
-
-Allocation IDs assign unique identifiers to shard copies. This allows the cluster to differentiate between multiple
-copies of the same data and track which shards have been active, so that after a cluster restart, shard copies
-containing only the most recent data can become primaries.
-
-=== Indices Shard Stores command
-
-By using allocation IDs instead of version numbers to identify shard copies for primary shard allocation, the former versioning scheme
-has become obsolete. This is reflected in the indices-shards-stores.html[Indices Shard Stores API]. A new field `allocation_id` replaces the
-former `version` field in the result of the Indices Shard Stores command. This field is available for all shard copies that have been either
-created with the current version of Elasticsearch or have been active in a cluster running a current version of Elasticsearch. For legacy
-shard copies that have not been active in a current version of Elasticsearch, a `legacy_version` field is available instead (equivalent to
-the former `version` field).
-
-=== Reroute commands
-
-The reroute command `allocate` has been split into two distinct commands `allocate_replica` and `allocate_empty_primary`.
-This was done as we introduced a new `allocate_stale_primary` command. The new `allocate_replica` command corresponds to the
-old `allocate` command  with `allow_primary` set to false. The new `allocate_empty_primary` command corresponds to the old
-`allocate` command with `allow_primary` set to true.
-
-==== `index.shared_filesystem.recover_on_any_node` changes
-
-The behavior of `index.shared_filesystem.recover_on_any_node = true` has been changed. Previously, in the case where no
-shard copies could be found, an arbitrary node was chosen by potentially ignoring allocation deciders. Now, we take
-balancing into account but don't assign the shard if the allocation deciders are not satisfied. The behavior has also changed
-in the case where shard copies can be found. Previously, a node not holding the shard copy was chosen if none of the nodes
-holding shard copies were satisfying the allocation deciders. Now, the shard will be assigned to a node having a shard copy,
-even if none of the nodes holding a shard copy satisfy the allocation deciders.
-
-[[breaking_50_percolator]]
-=== Percolator
-
-Adding percolator queries and modifications to existing percolator queries are no longer visible in immediately
-to the percolator. A refresh is required to run before the changes are visible to the percolator.
-
-The reason that this has changed is that on newly created indices the percolator automatically indexes the query terms
-and these query terms are used at percolate time to reduce the amount of queries the percolate API needs evaluate.
-This optimization didn't work in the percolate API mode where modifications to queries are immediately visible.
-
-The percolator by defaults sets the `size` option to `10` whereas before this was set to unlimited.
-
-The percolate api can no longer accept documents that have fields that don't exist in the mapping.
-
-When percolating an existing document then specifying a document in the source of the percolate request is not allowed
-any more.
-
-The percolate api no longer modifies the mappings. Before the percolate api could be used to dynamically introduce new
-fields to the mappings based on the fields in the document being percolated. This no longer works, because these
-unmapped fields are not persisted in the mapping.
-
-Percolator documents are no longer excluded from the search response.
-
-[[breaking_50_packaging]]
-=== Packaging
-
-==== Default logging using systemd (since Elasticsearch 2.2.0)
-
-In previous versions of Elasticsearch, the default logging
-configuration routed standard output to /dev/null and standard error to
-the journal. However, there are often critical error messages at
-startup that are logged to standard output rather than standard error
-and these error messages would be lost to the nether. The default has
-changed to now route standard output to the journal and standard error
-to inherit this setting (these are the defaults for systemd). These
-settings can be modified by editing the elasticsearch.service file.
-
-[[breaking_50_scripting]]
-=== Scripting
-
-==== Script mode settings
-
-Previously script mode settings (e.g., "script.inline: true",
-"script.engine.groovy.inline.aggs: false", etc.) accepted the values
-`on`, `true`, `1`, and `yes` for enabling a scripting mode, and the
-values `off`, `false`, `0`, and `no` for disabling a scripting mode.
-The variants `on`, `1`, and `yes ` for enabling and `off`, `0`,
-and `no` for disabling are no longer supported.
-
-==== Groovy dependencies
-
-In previous versions of Elasticsearch, the Groovy scripting capabilities
-depended on the `org.codehaus.groovy:groovy-all` artifact.  In addition
-to pulling in the Groovy language, this pulls in a very large set of
-functionality, none of which is needed for scripting within
-Elasticsearch. Aside from the inherent difficulties in managing such a
-large set of dependencies, this also increases the surface area for
-security issues. This dependency has been reduced to the core Groovy
-language `org.codehaus.groovy:groovy` artifact.
-
-[[breaking_50_term_vectors]]
-=== Term vectors
-
-The term vectors APIs no longer persist unmapped fields in the mappings.
-
-The `dfs` parameter has been removed completely, term vectors don't support
-distributed document frequencies anymore.
-
-[[breaking_50_security]]
-=== Security
-
-The option to disable the security manager `--security.manager.enabled` has been removed. In order to grant special
-permissions to elasticsearch users must tweak the local Java Security Policy.
diff --git a/docs/reference/migration/migrate_5_0/allocation.asciidoc b/docs/reference/migration/migrate_5_0/allocation.asciidoc
new file mode 100644
index 00000000000..1e095831381
--- /dev/null
+++ b/docs/reference/migration/migrate_5_0/allocation.asciidoc
@@ -0,0 +1,54 @@
+[[breaking_50_allocation]]
+=== Allocation changes
+
+==== Primary shard allocation
+
+Previously, primary shards were only assigned if a quorum of shard copies were
+found (configurable using `index.recovery.initial_shards`, now deprecated). In
+case where a primary had only a single replica, quorum was defined to be a
+single shard. This meant that any shard copy of an index with replication
+factor 1 could become primary, even it was a stale copy of the data on disk.
+This is now fixed thanks to shard allocation IDs.
+
+Allocation IDs assign unique identifiers to shard copies. This allows the
+cluster to differentiate between multiple copies of the same data and track
+which shards have been active so that, after a cluster restart, only shard
+copies containing the most recent data can become primaries.
+
+==== Indices Shard Stores command
+
+By using allocation IDs instead of version numbers to identify shard copies
+for primary shard allocation, the former versioning scheme has become
+obsolete. This is reflected in the
+<<indices-shards-stores,Indices Shard Stores API>>.
+
+A new `allocation_id` field replaces the former `version` field in the result
+of the Indices Shard Stores command. This field is available for all shard
+copies that have been either created with the current version of Elasticsearch
+or have been active in a cluster running a current version of Elasticsearch.
+For legacy shard copies that have not been active in a current version of
+Elasticsearch, a `legacy_version` field is available instead (equivalent to
+the former `version` field).
+
+==== Reroute commands
+
+The reroute command `allocate` has been split into two distinct commands
+`allocate_replica` and `allocate_empty_primary`. This was done as we
+introduced a new `allocate_stale_primary` command. The new `allocate_replica`
+command corresponds to the old `allocate` command  with `allow_primary` set to
+false. The new `allocate_empty_primary` command corresponds to the old
+`allocate` command with `allow_primary` set to true.
+
+==== `index.shared_filesystem.recover_on_any_node` changes
+
+The behavior of `index.shared_filesystem.recover_on_any_node: true` has been
+changed. Previously, in the case where no shard copies could be found, an
+arbitrary node was chosen by potentially ignoring allocation deciders. Now, we
+take balancing into account but don't assign the shard if the allocation
+deciders are not satisfied.
+
+The behavior has also changed in the case where shard copies can be found.
+Previously, a node not holding the shard copy was chosen if none of the nodes
+holding shard copies were satisfying the allocation deciders. Now, the shard
+will be assigned to a node having a shard copy, even if none of the nodes
+holding a shard copy satisfy the allocation deciders.
diff --git a/docs/reference/migration/migrate_5_0/cat.asciidoc b/docs/reference/migration/migrate_5_0/cat.asciidoc
new file mode 100644
index 00000000000..c3b1c84ee8d
--- /dev/null
+++ b/docs/reference/migration/migrate_5_0/cat.asciidoc
@@ -0,0 +1,33 @@
+[[breaking_50_cat_api]]
+=== CAT API changes
+
+==== Use Accept header for specifying response media type
+
+Previous versions of Elasticsearch accepted the Content-type header
+field for controlling the media type of the response in the cat API.
+This is in opposition to the HTTP spec which specifies the Accept
+header field for this purpose. Elasticsearch now uses the Accept header
+field and support for using the Content-Type header field for this
+purpose has been removed.
+
+==== Host field removed from the cat nodes API
+
+The `host` field has been removed from the cat nodes API as its value
+is always equal to the `ip` field. The `name` field is available in the
+cat nodes API and should be used instead of the `host` field.
+
+==== Changes to cat recovery API
+
+The fields `bytes_recovered` and `files_recovered` have been added to
+the cat recovery API. These fields, respectively, indicate the total
+number of bytes and files that have been recovered.
+
+The fields `total_files` and `total_bytes` have been renamed to
+`files_total` and `bytes_total`, respectively.
+
+Additionally, the field `translog` has been renamed to
+`translog_ops_recovered`, the field `translog_total` to
+`translog_ops` and the field `translog_percent` to
+`translog_ops_percent`. The short aliases for these fields are `tor`,
+`to`, and `top`, respectively.
+
diff --git a/docs/reference/migration/migrate_5_0/index-apis.asciidoc b/docs/reference/migration/migrate_5_0/index-apis.asciidoc
new file mode 100644
index 00000000000..b820194c2d0
--- /dev/null
+++ b/docs/reference/migration/migrate_5_0/index-apis.asciidoc
@@ -0,0 +1,51 @@
+[[breaking_50_index_apis]]
+=== Index APIs changes
+
+==== Closing / deleting indices while running snapshot
+
+In previous versions of Elasticsearch, closing or deleting an index during a
+full snapshot would make the snapshot fail. In 5.0, the close/delete index
+request will fail instead. The behavior for partial snapshots remains
+unchanged: Closing or deleting an index during a partial snapshot is still
+possible. The snapshot result is then marked as partial.
+
+==== Warmers
+
+Thanks to several changes like doc values by default and disk-based norms,
+warmers are no longer useful. As a consequence, warmers and the warmer API
+have been removed: it is no longer possible to register queries that will run
+before a new IndexSearcher is published.
+
+Don't worry if you have warmers defined on your indices, they will simply be
+ignored when upgrading to 5.0.
+
+==== System CPU stats
+
+The recent CPU usage (as a percent) has been added to the OS stats
+reported under the node stats API and the cat nodes API. The breaking
+change here is that there is a new object in the `os` object in the node
+stats response. This object is called `cpu` and includes percent` and
+`load_average` as fields. This moves the `load_average` field that was
+previously a top-level field in the `os` object to the `cpu` object. The
+format of the `load_average` field has changed to an object with fields
+`1m`, `5m`, and `15m` representing the one-minute, five-minute and
+fifteen-minute loads respectively. If any of these fields are not present,
+it indicates that the corresponding value is not available.
+
+In the cat nodes API response, the `cpu` field is output by default. The
+previous `load` field has been removed and is replaced by `load_1m`,
+`load_5m`, and `load_15m` which represent the one-minute, five-minute
+and fifteen-minute loads respectively. The field will be null if the
+corresponding value is not available.
+
+Finally, the API for `org.elasticsearch.monitor.os.OsStats` has
+changed. The `getLoadAverage` method has been removed. The value for
+this can now be obtained from `OsStats.Cpu#getLoadAverage` but it is no
+longer a double and is instead an object encapsulating the one-minute,
+five-minute and fifteen-minute load averages. Additionally, the recent
+CPU usage can be obtained from `OsStats.Cpu#getPercent`.
+
+==== Suggest stats
+
+Suggest stats exposed through `suggest` in indices stats has been merged
+with `search` stats. `suggest` stats is exposed as part of `search` stats.
diff --git a/docs/reference/migration/migrate_5_0/java.asciidoc b/docs/reference/migration/migrate_5_0/java.asciidoc
new file mode 100644
index 00000000000..d7c580bd008
--- /dev/null
+++ b/docs/reference/migration/migrate_5_0/java.asciidoc
@@ -0,0 +1,306 @@
+
+
+
+[[breaking_50_java_api_changes]]
+=== Java API changes
+
+==== Count api has been removed
+
+The deprecated count api has been removed from the Java api, use the search api instead and set size to 0.
+
+The following call
+
+[source,java]
+-----
+client.prepareCount(indices).setQuery(query).get();
+-----
+
+can be replaced with
+
+[source,java]
+-----
+client.prepareSearch(indices).setSource(new SearchSourceBuilder().size(0).query(query)).get();
+-----
+
+==== Suggest api has been removed
+
+The suggest api has been removed from the Java api, use the suggest option in search api, it has been optimized
+for suggest-only request.
+
+The following call
+
+[source,java]
+-----
+client.prepareSuggest(indices).addSuggestion("foo", SuggestBuilders.completionSuggestion("field").text("s")).get();
+-----
+
+can be replaced with
+
+[source,java]
+-----
+client.prepareSearch(indices).suggest(new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion("field").text("s"))).get();
+-----
+
+==== Elasticsearch will no longer detect logging implementations
+
+Elasticsearch now logs only to log4j 1.2. Previously if log4j wasn't on the
+classpath it made some effort to degrade to slf4j or java.util.logging. Now it
+will fail to work without the log4j 1.2 api. The log4j-over-slf4j bridge ought
+to work when using the java client, as should log4j 2's log4j-1.2-api. The
+Elasticsearch server now only supports log4j as configured by `logging.yml`
+and will fail if log4j isn't present.
+
+==== Groovy dependencies
+
+In previous versions of Elasticsearch, the Groovy scripting capabilities
+depended on the `org.codehaus.groovy:groovy-all` artifact.  In addition
+to pulling in the Groovy language, this pulls in a very large set of
+functionality, none of which is needed for scripting within
+Elasticsearch. Aside from the inherent difficulties in managing such a
+large set of dependencies, this also increases the surface area for
+security issues. This dependency has been reduced to the core Groovy
+language `org.codehaus.groovy:groovy` artifact.
+
+==== DocumentAlreadyExistsException removed
+
+`DocumentAlreadyExistsException` is removed and a `VersionConflictException` is thrown instead (with a better
+error description). This will influence code that use the `IndexRequest.opType()` or `IndexRequest.create()`
+to index a document only if it doesn't already exist.
+
+==== Changes to Query Builders
+
+===== BoostingQueryBuilder
+
+Removed setters for mandatory positive/negative query. Both arguments now have
+to be supplied at construction time already and have to be non-null.
+
+===== SpanContainingQueryBuilder
+
+Removed setters for mandatory big/little inner span queries. Both arguments now have
+to be supplied at construction time already and have to be non-null. Updated
+static factory methods in QueryBuilders accordingly.
+
+===== SpanOrQueryBuilder
+
+Making sure that query contains at least one clause by making initial clause mandatory
+in constructor.
+
+===== SpanNearQueryBuilder
+
+Removed setter for mandatory slop parameter, needs to be set in constructor now. Also
+making sure that query contains at least one clause by making initial clause mandatory
+in constructor. Updated the static factory methods in QueryBuilders accordingly.
+
+===== SpanNotQueryBuilder
+
+Removed setter for mandatory include/exclude span query clause, needs to be set in constructor now.
+Updated the static factory methods in QueryBuilders and tests accordingly.
+
+===== SpanWithinQueryBuilder
+
+Removed setters for mandatory big/little inner span queries. Both arguments now have
+to be supplied at construction time already and have to be non-null. Updated
+static factory methods in QueryBuilders accordingly.
+
+===== QueryFilterBuilder
+
+Removed the setter `queryName(String queryName)` since this field is not supported
+in this type of query. Use `FQueryFilterBuilder.queryName(String queryName)` instead
+when in need to wrap a named query as a filter.
+
+===== WrapperQueryBuilder
+
+Removed `wrapperQueryBuilder(byte[] source, int offset, int length)`. Instead simply
+use  `wrapperQueryBuilder(byte[] source)`. Updated the static factory methods in
+QueryBuilders accordingly.
+
+===== QueryStringQueryBuilder
+
+Removed ability to pass in boost value using `field(String field)` method in form e.g. `field^2`.
+Use the `field(String, float)` method instead.
+
+===== Operator
+
+Removed the enums called `Operator` from `MatchQueryBuilder`, `QueryStringQueryBuilder`,
+`SimpleQueryStringBuilder`, and `CommonTermsQueryBuilder` in favour of using the enum
+defined in `org.elasticsearch.index.query.Operator` in an effort to consolidate the
+codebase and avoid duplication.
+
+===== queryName and boost support
+
+Support for `queryName` and `boost` has been streamlined to all of the queries. That is
+a breaking change till queries get sent over the network as serialized json rather
+than in `Streamable` format. In fact whenever additional fields are added to the json
+representation of the query, older nodes might throw error when they find unknown fields.
+
+===== InnerHitsBuilder
+
+InnerHitsBuilder now has a dedicated addParentChildInnerHits and addNestedInnerHits methods
+to differentiate between inner hits for nested vs. parent / child documents. This change
+makes the type / path parameter mandatory.
+
+===== MatchQueryBuilder
+
+Moving MatchQueryBuilder.Type and MatchQueryBuilder.ZeroTermsQuery enum to MatchQuery.Type.
+Also reusing new Operator enum.
+
+===== MoreLikeThisQueryBuilder
+
+Removed `MoreLikeThisQueryBuilder.Item#id(String id)`, `Item#doc(BytesReference doc)`,
+`Item#doc(XContentBuilder doc)`. Use provided constructors instead.
+
+Removed `MoreLikeThisQueryBuilder#addLike` in favor of texts and/or items being provided
+at construction time. Using arrays there instead of lists now.
+
+Removed `MoreLikeThisQueryBuilder#addUnlike` in favor to using the `unlike` methods
+which take arrays as arguments now rather than the lists used before.
+
+The deprecated `docs(Item... docs)`, `ignoreLike(Item... docs)`,
+`ignoreLike(String... likeText)`, `addItem(Item... likeItems)` have been removed.
+
+===== GeoDistanceQueryBuilder
+
+Removing individual setters for lon() and lat() values, both values should be set together
+ using point(lon, lat).
+
+===== GeoDistanceRangeQueryBuilder
+
+Removing setters for to(Object ...) and from(Object ...) in favour of the only two allowed input
+arguments (String, Number). Removing setter for center point (point(), geohash()) because parameter
+is mandatory and should already be set in constructor.
+Also removing setters for lt(), lte(), gt(), gte() since they can all be replaced by equivalent
+calls to to/from() and inludeLower()/includeUpper().
+
+===== GeoPolygonQueryBuilder
+
+Require shell of polygon already to be specified in constructor instead of adding it pointwise.
+This enables validation, but makes it necessary to remove the addPoint() methods.
+
+===== MultiMatchQueryBuilder
+
+Moving MultiMatchQueryBuilder.ZeroTermsQuery enum to MatchQuery.ZeroTermsQuery.
+Also reusing new Operator enum.
+
+Removed ability to pass in boost value using `field(String field)` method in form e.g. `field^2`.
+Use the `field(String, float)` method instead.
+
+===== MissingQueryBuilder
+
+The MissingQueryBuilder which was deprecated in 2.2.0 is removed. As a replacement use ExistsQueryBuilder
+inside a mustNot() clause. So instead of using `new ExistsQueryBuilder(name)` now use
+`new BoolQueryBuilder().mustNot(new ExistsQueryBuilder(name))`.
+
+===== NotQueryBuilder
+
+The NotQueryBuilder which was deprecated in 2.1.0 is removed. As a replacement use BoolQueryBuilder
+with added mustNot() clause. So instead of using `new NotQueryBuilder(filter)` now use
+`new BoolQueryBuilder().mustNot(filter)`.
+
+===== TermsQueryBuilder
+
+Remove the setter for `termsLookup()`, making it only possible to either use a TermsLookup object or
+individual values at construction time. Also moving individual settings for the TermsLookup (lookupIndex,
+lookupType, lookupId, lookupPath) to the separate TermsLookup class, using constructor only and moving
+checks for validation there. Removed `TermsLookupQueryBuilder` in favour of `TermsQueryBuilder`.
+
+===== FunctionScoreQueryBuilder
+
+`add` methods have been removed, all filters and functions must be provided as constructor arguments by
+creating an array of `FunctionScoreQueryBuilder.FilterFunctionBuilder` objects, containing one element
+for each filter/function pair.
+
+`scoreMode` and `boostMode` can only be provided using corresponding enum members instead
+of string values: see `FilterFunctionScoreQuery.ScoreMode` and `CombineFunction`.
+
+`CombineFunction.MULT` has been renamed to `MULTIPLY`.
+
+===== IdsQueryBuilder
+
+For simplicity, only one way of adding the ids to the existing list (empty by default)  is left: `addIds(String...)`
+
+===== ShapeBuilders
+
+`InternalLineStringBuilder` is removed in favour of `LineStringBuilder`, `InternalPolygonBuilder` in favour of PolygonBuilder` and `Ring` has been replaced with `LineStringBuilder`. Also the abstract base classes `BaseLineStringBuilder` and `BasePolygonBuilder` haven been merged with their corresponding implementations.
+
+===== RescoreBuilder
+
+`RecoreBuilder.Rescorer` was merged with `RescoreBuilder`, which now is an abstract superclass. QueryRescoreBuilder currently is its only implementation.
+
+===== PhraseSuggestionBuilder
+
+The inner DirectCandidateGenerator class has been moved out to its own class called DirectCandidateGeneratorBuilder.
+
+===== SortBuilders
+
+The `sortMode` setter in `FieldSortBuilder`, `GeoDistanceSortBuilder` and `ScriptSortBuilder` now
+accept a `SortMode` enum instead of a String constant. Also the getter returns the same enum type.
+
+===== SuggestBuilder
+
+The `setText` method has been changed to `setGlobalText` to make the intent more clear, and a `getGlobalText` method has been added.
+
+The `addSuggestion` method now required the user specified suggestion name, previously used in the ctor of each suggestion.
+
+===== SuggestionBuilder
+
+The `field` setter has been deleted. Instead the field name needs to be specified as constructor argument.
+
+==== SearchSourceBuilder
+
+All methods which take an `XContentBuilder`, `BytesReference` `Map<String, Object>` or `bytes[]` have been removed in favor of providing the
+relevant builder object for that feature (e.g. `HighlightBuilder`, `AggregationBuilder`, `SuggestBuilder`) . This means that all search requests
+can now be validated at call time which results in much clearer errors.
+
+The `defaultResourceWindowSize(int)` method has been removed. The window size should be set explicitly on all `RescoreBuilder` objects.
+
+==== SearchRequestBuilder
+
+All methods which take an `XContentBuilder`, `BytesReference` `Map<String, Object>` or `bytes[]` have been removed in favor of providing the
+relevant builder object for that feature (e.g. `HighlightBuilder`, `AggregationBuilder`, `SuggestBuilder`) . This means that all search requests
+can now be validated at call time which results in much clearer errors.
+
+All highlighter methods have been removed in favor of a single `highlighter(HighlightBuilder)` method.
+
+The `setExtraSource(SearchSourceBuilder)` method has been removed.
+
+The `setTemplateSource(String)` and `setTemplateSource(BytesReference)` methods have been removed. Use `setTemplate(Template)` instead.
+
+`setRescorer(Rescorer)` and `setRescorer(Rescorer, int)` have been removed infavor of `setRescorer(RescoreBuilder)` and `setRescorer(RescoreBuilder, int)`
+
+==== SearchRequest
+
+All `template` methods have been removed in favor of a single `template(Template)` method.
+
+All `source` methods have been removed in favor of a single `source(SearchSourceBuilder)` method. This means that all search requests can now be validated
+at call time which results in much clearer errors.
+
+All `extraSource` methods have been removed.
+
+==== AggregationBuilder
+
+All methods which take an `XContentBuilder`, `BytesReference` `Map<String, Object>` or `bytes[]` have been removed in favor of providing the
+relevant builder object (i.e. `subAggregation(AggregationBuilder)` or `subAggregation(PipelineAggregationBuilder)`). This means that all
+requests can now be validated at call time which results in much clearer errors.
+
+==== ValidateQueryRequest
+
+`source(QuerySourceBuilder)`, `source(Map)`, `source(XContentBuilder)`, `source(String)`, `source(byte[])`, `source(byte[], int, int)`,
+`source(BytesReference)` and `source()` have been removed in favor of using `query(QueryBuilder<?>)` and `query()`
+
+==== ValidateQueryRequestBuilder
+
+`setSource()` methods have been removed in favor of using `setQuery(QueryBuilder<?>)`
+
+==== ExplainRequest
+
+`source(QuerySourceBuilder)`, `source(Map)`, `source(BytesReference)` and `source()` have been removed in favor of using
+`query(QueryBuilder<?>)` and `query()`
+
+==== ExplainRequestBuilder
+
+The `setQuery(BytesReference)` method have been removed in favor of using `setQuery(QueryBuilder<?>)`
+
+=== ClusterStatsResponse
+
+Removed the `getMemoryAvailable` method from `OsStats`, which could be previously accessed calling
+`clusterStatsResponse.getNodesStats().getOs().getMemoryAvailable()`.
diff --git a/docs/reference/migration/migrate_5_0/mapping.asciidoc b/docs/reference/migration/migrate_5_0/mapping.asciidoc
new file mode 100644
index 00000000000..23298cd733c
--- /dev/null
+++ b/docs/reference/migration/migrate_5_0/mapping.asciidoc
@@ -0,0 +1,112 @@
+[[breaking_50_mapping_changes]]
+=== Mapping changes
+
+==== `string` fields replaced by `text`/`keyword` fields
+
+The `string` field datatype has been replaced by the `text` field for full
+text analyzed content, and the `keyword` field for not-analyzed exact string
+values.  For backwards compatibility purposes, during the 5.x series:
+
+* `string` fields on pre-5.0 indices will function as before.
+* New `string` fields can be added to pre-5.0 indices as before.
+* `text` and `keyword` fields can also be added to pre-5.0 indices.
+* When adding a `string` field to a new index, the field mapping will be
+  rewritten as a `text` or `keyword` field if possible, otherwise
+  an exception will be thrown.  Certain configurations that were possible
+  with `string` fields are no longer possible with `text`/`keyword` fields
+  such as enabling `term_vectors` on a not-analyzed `keyword` field.
+
+==== Default string mappings
+
+String mappings now have the following default mappings:
+
+[source,json]
+---------------
+{
+  "type": "text",
+  "fields": {
+    "keyword": {
+      "type": "keyword",
+      "ignore_above": 256
+    }
+  }
+}
+---------------
+
+This allows to perform full-text search on the original field name and to sort
+and run aggregations on the sub keyword field.
+
+==== `index` property
+
+On all field datatypes (except for the deprecated `string` field), the `index`
+property now only accepts `true`/`false` instead of `not_analyzed`/`no`. The
+`string` field still accepts `analyzed`/`not_analyzed`/`no`.
+
+==== Doc values on unindexed fields
+
+Previously, setting a field to `index:no` would also disable doc-values.  Now,
+doc-values are always enabled on numeric and boolean fields unless
+`doc_values` is set to `false`.
+
+==== Floating points use `float` instead of `double`
+
+When dynamically mapping a field containing a floating point number, the field
+now defaults to using `float` instead of `double`. The reasoning is that
+floats should be more than enough for most cases but would decrease storage
+requirements significantly.
+
+==== `norms`
+
+`norms` now take a boolean instead of an object. This boolean is the replacement
+for `norms.enabled`. There is no replacement for `norms.loading` since eager
+loading of norms is not useful anymore now that norms are disk-based.
+
+==== `fielddata.format`
+
+Setting `fielddata.format: doc_values` in the mappings used to implicitly
+enable doc-values on a field. This no longer works: the only way to enable or
+disable doc-values is by using the `doc_values` property of mappings.
+
+==== `fielddata.frequency.regex`
+
+Regex filters are not supported anymore and will be dropped on upgrade.
+
+==== Source-transform removed
+
+The source `transform` feature has been removed. Instead, use an ingest pipeline
+
+==== `_parent` field no longer indexed
+
+The join between parent and child documents no longer relies on indexed fields
+and therefore from 5.0.0 onwards the `_parent` field is no longer indexed. In
+order to find documents that referrer to a specific parent id the new
+`parent_id` query can be used. The GET response and hits inside the search
+response still include the parent id under the `_parent` key.
+
+==== Source `format` option
+
+The `_source` mapping no longer supports the `format` option. It will still be
+accepted for indices created before the upgrade to 5.0 for backwards
+compatibility, but it will have no effect. Indices created on or after 5.0
+will reject this option.
+
+==== Object notation
+
+Core types no longer support the object notation, which was used to provide
+per document boosts as follows:
+
+[source,json]
+---------------
+{
+  "value": "field_value",
+  "boost": 42
+}
+---------------
+
+==== Boost accuracy for queries on `_all`
+
+Per-field boosts on the `_all` are now compressed into a single byte instead
+of the 4 bytes used previously. While this will make the index much more
+space-efficient, it also means that index time boosts will be less accurately
+encoded.
+
diff --git a/docs/reference/migration/migrate_5_0/packaging.asciidoc b/docs/reference/migration/migrate_5_0/packaging.asciidoc
new file mode 100644
index 00000000000..9be2d4accac
--- /dev/null
+++ b/docs/reference/migration/migrate_5_0/packaging.asciidoc
@@ -0,0 +1,24 @@
+[[breaking_50_packaging]]
+=== Packaging
+
+==== Default logging using systemd (since Elasticsearch 2.2.0)
+
+In previous versions of Elasticsearch, the default logging
+configuration routed standard output to /dev/null and standard error to
+the journal. However, there are often critical error messages at
+startup that are logged to standard output rather than standard error
+and these error messages would be lost to the nether. The default has
+changed to now route standard output to the journal and standard error
+to inherit this setting (these are the defaults for systemd). These
+settings can be modified by editing the elasticsearch.service file.
+
+==== Longer startup times
+
+In Elasticsearch 5.0.0 the `-XX:+AlwaysPreTouch` flag has been added to the JVM
+startup options. This option touches all memory pages used by the JVM heap
+during initialization of the HotSpot VM to reduce the chance of having to commit
+a memory page during GC time. This will increase the startup time of
+Elasticsearch as well as increasing the initial resident memory usage of the
+Java process.
+
+
diff --git a/docs/reference/migration/migrate_5_0/percolator.asciidoc b/docs/reference/migration/migrate_5_0/percolator.asciidoc
new file mode 100644
index 00000000000..73f262afb70
--- /dev/null
+++ b/docs/reference/migration/migrate_5_0/percolator.asciidoc
@@ -0,0 +1,43 @@
+[[breaking_50_percolator]]
+=== Percolator changes
+
+==== Percolator is near-real time
+
+Previously percolators were activated in real-time, i.e. as soon as they were
+indexed.  Now, changes to the percolator query are visible in near-real time,
+as soon as the index has been refreshed. This change was required because, in
+indices created from 5.0 onwards, the terms used in a percolator query are
+automatically indexed to allow for more efficient query selection during
+percolation.
+
+==== Percolate and multi percolator APIs
+
+Percolator and multi percolate APIs have been deprecated and will be removed in the next major release. These APIs have
+been replaced by the `percolator` query that can be used in the search and multi search APIs.
+
+==== Percolator mapping
+
+The `percolator` query  can no longer accept documents that reference fields
+that don't already exist in the mapping. Before the percolate API allowed this.
+
+The `percolator` query no longer modifies the mappings. Before the percolate API
+could be used to dynamically introduce new fields to the mappings based on the
+fields in the document being percolated. This no longer works, because these
+unmapped fields are not persisted in the mapping.
+
+==== Percolator documents returned by search
+
+Documents with the `.percolate` type were previously excluded from the search
+response, unless the `.percolate` type was specified explicitly in the search
+request.  Now, percolator documents are treated in the same way as any other
+document and are returned by search requests.
+
+==== Percolating existing document
+
+When percolating an existing document then also specifying a document as source in the
+`percolator` query is not allowed any more. Before the percolate API allowed and ignored
+the existing document.
+
+==== Percolate Stats
+
+Percolate stats have been replaced with percolator query cache stats in nodes stats and cluster stats APIs.
\ No newline at end of file
diff --git a/docs/reference/migration/migrate_5_0/plugins.asciidoc b/docs/reference/migration/migrate_5_0/plugins.asciidoc
new file mode 100644
index 00000000000..46014e3bcd4
--- /dev/null
+++ b/docs/reference/migration/migrate_5_0/plugins.asciidoc
@@ -0,0 +1,103 @@
+[[breaking_50_plugins]]
+=== Plugin changes
+
+The command `bin/plugin` has been renamed to `bin/elasticsearch-plugin`. The
+structure of the plugin ZIP archive has changed. All the plugin files must be
+contained in a top-level directory called `elasticsearch`. If you use the
+gradle build, this structure is automatically generated.
+
+==== Plugins isolation
+
+`isolated` option has been removed. Each plugin will have its own classloader.
+
+==== Site plugins removed
+
+Site plugins have been removed. Site plugins should be reimplemented as Kibana
+plugins.
+
+==== Multicast plugin removed
+
+Multicast has been removed. Use unicast discovery, or one of the cloud
+discovery plugins.
+
+==== Plugins with custom query implementations
+
+Plugins implementing custom queries need to implement the `fromXContent(QueryParseContext)` method in their
+`QueryParser` subclass rather than `parse`. This method will take care of parsing the query from `XContent` format
+into an intermediate query representation that can be streamed between the nodes in binary format, effectively the
+query object used in the java api. Also, the query parser needs to implement the `getBuilderPrototype` method that
+returns a prototype of the `NamedWriteable` query, which allows to deserialize an incoming query by calling
+`readFrom(StreamInput)` against it, which will create a new object, see usages of `Writeable`. The `QueryParser`
+also needs to declare the generic type of the query that it supports and it's able to parse.
+The query object can then transform itself into a lucene query through the new `toQuery(QueryShardContext)` method,
+which returns a lucene query to be executed on the data node.
+
+Similarly, plugins implementing custom score functions need to implement the `fromXContent(QueryParseContext)`
+method in their `ScoreFunctionParser` subclass rather than `parse`. This method will take care of parsing
+the function from `XContent` format into an intermediate function representation that can be streamed between
+the nodes in binary format, effectively the function object used in the java api. Also, the query parser needs
+to implement the `getBuilderPrototype` method that returns a prototype of the `NamedWriteable` function, which
+allows to deserialize an incoming function by calling `readFrom(StreamInput)` against it, which will create a
+new object, see usages of `Writeable`. The `ScoreFunctionParser` also needs to declare the generic type of the
+function that it supports and it's able to parse. The function object can then transform itself into a lucene
+function through the new `toFunction(QueryShardContext)` method, which returns a lucene function to be executed
+on the data node.
+
+==== Cloud AWS plugin changes
+
+Cloud AWS plugin has been split in two plugins:
+
+* {plugins}/discovery-ec2.html[Discovery EC2 plugin]
+* {plugins}/repository-s3.html[Repository S3 plugin]
+
+Proxy settings for both plugins have been renamed:
+
+* from `cloud.aws.proxy_host` to `cloud.aws.proxy.host`
+* from `cloud.aws.ec2.proxy_host` to `cloud.aws.ec2.proxy.host`
+* from `cloud.aws.s3.proxy_host` to `cloud.aws.s3.proxy.host`
+* from `cloud.aws.proxy_port` to `cloud.aws.proxy.port`
+* from `cloud.aws.ec2.proxy_port` to `cloud.aws.ec2.proxy.port`
+* from `cloud.aws.s3.proxy_port` to `cloud.aws.s3.proxy.port`
+
+==== Cloud Azure plugin changes
+
+Cloud Azure plugin has been split in three plugins:
+
+* {plugins}/discovery-azure.html[Discovery Azure plugin]
+* {plugins}/repository-azure.html[Repository Azure plugin]
+* {plugins}/store-smb.html[Store SMB plugin]
+
+If you were using the `cloud-azure` plugin for snapshot and restore, you had in `elasticsearch.yml`:
+
+[source,yaml]
+-----
+cloud:
+    azure:
+        storage:
+            account: your_azure_storage_account
+            key: your_azure_storage_key
+-----
+
+You need to give a unique id to the storage details now as you can define multiple storage accounts:
+
+[source,yaml]
+-----
+cloud:
+    azure:
+        storage:
+            my_account:
+                account: your_azure_storage_account
+                key: your_azure_storage_key
+-----
+
+
+==== Cloud GCE plugin changes
+
+Cloud GCE plugin has been renamed to {plugins}/discovery-gce.html[Discovery GCE plugin].
+
+
+==== Mapper Attachments plugin deprecated
+
+Mapper attachments has been deprecated. Users should use now the {plugins}/ingest-attachment.html[`ingest-attachment`]
+plugin.
+
diff --git a/docs/reference/migration/migrate_5_0/rest.asciidoc b/docs/reference/migration/migrate_5_0/rest.asciidoc
new file mode 100644
index 00000000000..0b8d76ea10f
--- /dev/null
+++ b/docs/reference/migration/migrate_5_0/rest.asciidoc
@@ -0,0 +1,22 @@
+
+[[breaking_50_rest_api_changes]]
+=== REST API changes
+
+==== id values longer than 512 bytes are rejected
+
+When specifying an `_id` value longer than 512 bytes, the request will be
+rejected.
+
+==== `/_optimize` endpoint removed
+
+The deprecated `/_optimize` endpoint has been removed. The `/_forcemerge`
+endpoint should be used in lieu of optimize.
+
+The `GET` HTTP verb for `/_forcemerge` is no longer supported, please use the
+`POST` HTTP verb.
+
+==== Removed `mem` section from `/_cluster/stats` response
+
+The `mem` section contained only one value, the total memory available
+throughout all nodes in the cluster. The section was removed as it didn't
+prove useful.
diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
new file mode 100644
index 00000000000..595cf2e3fe9
--- /dev/null
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -0,0 +1,143 @@
+[[breaking_50_search_changes]]
+=== Search and Query DSL changes
+
+==== `search_type`
+
+===== `search_type=count` removed
+
+The `count` search type was deprecated since version 2.0.0 and is now removed.
+In order to get the same benefits, you just need to set the value of the `size`
+parameter to `0`.
+
+For instance, the following request:
+
+[source,sh]
+---------------
+GET /my_index/_search?search_type=count
+{
+  "aggs": {
+    "my_terms": {
+       "terms": {
+         "field": "foo"
+       }
+     }
+  }
+}
+---------------
+
+can be replaced with:
+
+[source,sh]
+---------------
+GET /my_index/_search
+{
+  "size": 0,
+  "aggs": {
+    "my_terms": {
+       "terms": {
+         "field": "foo"
+       }
+     }
+  }
+}
+---------------
+
+===== `search_type=scan` removed
+
+The `scan` search type was deprecated since version 2.1.0 and is now removed.
+All benefits from this search type can now be achieved by doing a scroll
+request that sorts documents in `_doc` order, for instance:
+
+[source,sh]
+---------------
+GET /my_index/_search?scroll=2m
+{
+  "sort": [
+    "_doc"
+  ]
+}
+---------------
+
+Scroll requests sorted by `_doc` have been optimized to more efficiently resume
+from where the previous request stopped, so this will have the same performance
+characteristics as the former `scan` search type.
+
+==== `fields` parameter
+
+The `fields` parameter used to try to retrieve field values from stored
+fields, and fall back to extracting from the `_source` if a field is not
+marked as stored.  Now, the `fields` parameter will only return stored fields
+-- it will no longer extract values from the `_source`.
+
+==== search-exists API removed
+
+The search exists api has been removed in favour of using the search api with
+`size` set to `0` and `terminate_after` set to `1`.
+
+
+==== Deprecated queries removed
+
+The following deprecated queries have been removed:
+
+`filtered`::      Use `bool` query instead, which supports `filter` clauses too.
+`and`::           Use `must` clauses in a `bool` query instead.
+`or`::            Use `should` clauses in a `bool` query instead.
+`limit`::         Use the `terminate_after` parameter instead.
+`fquery`::        Is obsolete after filters and queries have been merged.
+`query`::         Is obsolete after filters and queries have been merged.
+`query_binary`::  Was undocumented and has been removed.
+`filter_binary`:: Was undocumented and has been removed.
+
+
+==== Changes to queries
+
+* Removed support for the deprecated `min_similarity` parameter in `fuzzy
+  query`, in favour of `fuzziness`.
+
+* Removed support for the deprecated `fuzzy_min_sim` parameter in
+  `query_string` query, in favour of `fuzziness`.
+
+* Removed support for the deprecated `edit_distance` parameter in completion
+  suggester, in favour of `fuzziness`.
+
+* Removed support for the deprecated `filter` and `no_match_filter` fields in `indices` query,
+in favour of `query` and `no_match_query`.
+
+* Removed support for the deprecated `filter` fields in `nested` query, in favour of `query`.
+
+* Removed support for the deprecated `minimum_should_match` and
+  `disable_coord` in `terms` query, use `bool` query instead. Also removed
+  support for the deprecated `execution` parameter.
+
+* Removed support for the top level `filter` element in `function_score` query, replaced by `query`.
+
+* The `collect_payloads` parameter of the `span_near` query has been deprecated.  Payloads will be loaded when needed.
+
+* The `score_type` parameter to the `nested`, has_child` and `has_parent` queries has been removed in favour of `score_mode`.
+  Also, the `total` score mode has been removed in favour of the `sum` mode.
+
+* When the `max_children` parameter was set to `0` on the `has_child` query
+  then there was no upper limit on how many child documents were allowed to
+  match. Now, `0` really means that zero child documents are allowed. If no
+  upper limit is needed then the `max_children` parameter shouldn't be specified
+  at all.
+
+* The `exists` query will now fail if the `_field_names` field is disabled.
+
+
+==== Top level `filter` parameter
+
+Removed support for the deprecated top level `filter` in the search api,
+replaced by `post_filter`.
+
+==== Highlighters
+
+Removed support for multiple highlighter names, the only supported ones are:
+`plain`, `fvh` and `postings`.
+
+==== Term vectors API
+
+The term vectors APIs no longer persist unmapped fields in the mappings.
+
+The `dfs` parameter to the term vectors API has been removed completely. Term
+vectors don't support distributed document frequencies anymore.
diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc
new file mode 100644
index 00000000000..5e524a6a3cc
--- /dev/null
+++ b/docs/reference/migration/migrate_5_0/settings.asciidoc
@@ -0,0 +1,220 @@
+[[breaking_50_settings_changes]]
+=== Settings changes
+
+From Elasticsearch 5.0 on all settings are validated before they are applied.
+Node level and default index level settings are validated on node startup,
+dynamic cluster and index setting are validated before they are updated/added
+to the cluster state.
+
+Every setting must be a *known* setting. All settings must have been
+registered with the node or transport client they are used with. This implies
+that plugins that define custom settings must register all of their settings
+during plugin loading using the `SettingsModule#registerSettings(Setting)`
+method.
+
+==== Index Level Settings
+
+In previous versions Elasticsearch allowed to specify index level setting
+as _defaults_ on the node level, inside the `elasticsearch.yaml` file or even via
+command-line parameters. From Elasticsearch 5.0 on only selected settings like
+for instance `index.codec` can be set on the node level. All other settings must be
+set on each individual index. To set default values on every index, index templates
+should be used instead.
+
+==== Node settings
+
+The `name` setting has been removed and is replaced by `node.name`. Usage of
+`-Dname=some_node_name` is not supported anymore.
+
+==== Gateway settings
+
+The `gateway.format` setting for configuring global and index state serialization
+format has been removed. By default, `smile` is used as the format.
+
+==== Transport Settings
+
+All settings with a `netty` infix have been replaced by their already existing
+`transport` synonyms. For instance `transport.netty.bind_host` is no longer
+supported and should be replaced by the superseding setting
+`transport.bind_host`.
+
+==== Script mode settings
+
+Previously script mode settings (e.g., "script.inline: true",
+"script.engine.groovy.inline.aggs: false", etc.) accepted the values
+`on`, `true`, `1`, and `yes` for enabling a scripting mode, and the
+values `off`, `false`, `0`, and `no` for disabling a scripting mode.
+The variants `on`, `1`, and `yes ` for enabling and `off`, `0`,
+and `no` for disabling are no longer supported.
+
+
+==== Security manager settings
+
+The option to disable the security manager `security.manager.enabled` has been
+removed. In order to grant special permissions to elasticsearch users must
+edit the local Java Security Policy.
+
+==== Network settings
+
+The `_non_loopback_` value for settings like `network.host` would arbitrarily
+pick the first interface not marked as loopback. Instead, specify by address
+scope (e.g. `_local_,_site_` for all loopback and private network addresses)
+or by explicit interface names, hostnames, or addresses.
+
+==== Forbid changing of thread pool types
+
+Previously, <<modules-threadpool,thread pool types>> could be dynamically
+adjusted. The thread pool type effectively controls the backing queue for the
+thread pool and modifying this is an expert setting with minimal practical
+benefits and high risk of being misused. The ability to change the thread pool
+type for any thread pool has been removed. It is still possible to adjust
+relevant thread pool parameters for each of the thread pools (e.g., depending
+on the thread pool type, `keep_alive`, `queue_size`, etc.).
+
+==== Threadpool settings
+
+The `suggest` threadpool has been removed, now suggest requests use the
+`search` threadpool.
+
+==== Analysis settings
+
+The `index.analysis.analyzer.default_index` analyzer is not supported anymore.
+If you wish to change the analyzer to use for indexing, change the
+`index.analysis.analyzer.default` analyzer instead.
+
+==== Ping settings
+
+Previously, there were three settings for the ping timeout:
+`discovery.zen.initial_ping_timeout`, `discovery.zen.ping.timeout` and
+`discovery.zen.ping_timeout`. The former two have been removed and the only
+setting key for the ping timeout is now `discovery.zen.ping_timeout`. The
+default value for ping timeouts remains at three seconds.
+
+
+`discovery.zen.master_election.filter_client` and `discovery.zen.master_election.filter_data` have
+been removed in favor of the new `discovery.zen.master_election.ignore_non_master_pings`. This setting control how ping responses
+are interpreted during master election and should be used with care and only in extreme cases. See documentation for details.
+
+==== Recovery settings
+
+Recovery settings deprecated in 1.x have been removed:
+
+ * `index.shard.recovery.translog_size` is superseded by `indices.recovery.translog_size`
+ * `index.shard.recovery.translog_ops` is superseded by `indices.recovery.translog_ops`
+ * `index.shard.recovery.file_chunk_size` is superseded by `indices.recovery.file_chunk_size`
+ * `index.shard.recovery.concurrent_streams` is superseded by `indices.recovery.concurrent_streams`
+ * `index.shard.recovery.concurrent_small_file_streams` is superseded by `indices.recovery.concurrent_small_file_streams`
+ * `indices.recovery.max_size_per_sec` is superseded by `indices.recovery.max_bytes_per_sec`
+
+If you are using any of these settings please take the time to review their
+purpose. All of the settings above are considered _expert settings_ and should
+only be used if absolutely necessary. If you have set any of the above setting
+as persistent cluster settings please use the settings update API and set
+their superseded keys accordingly.
+
+The following settings have been removed without replacement
+
+ * `indices.recovery.concurrent_small_file_streams` - recoveries are now single threaded. The number of concurrent outgoing recoveries are throttled via allocation deciders
+ * `indices.recovery.concurrent_file_streams` - recoveries are now single threaded. The number of concurrent outgoing recoveries are throttled via allocation deciders
+
+==== Translog settings
+
+The `index.translog.flush_threshold_ops` setting is not supported anymore. In
+order to control flushes based on the transaction log growth use
+`index.translog.flush_threshold_size` instead.
+
+Changing the translog type with `index.translog.fs.type` is not supported
+anymore, the `buffered` implementation is now the only available option and
+uses a fixed `8kb` buffer.
+
+The translog by default is fsynced after every `index`, `create`, `update`,
+`delete`, or `bulk` request.  The ability to fsync on every operation is not
+necessary anymore. In fact, it can be a performance bottleneck and it's trappy
+since it enabled by a special value set on `index.translog.sync_interval`.
+Now, `index.translog.sync_interval`  doesn't accept a value less than `100ms`
+which prevents fsyncing too often if async durability is enabled. The special
+value `0` is no longer supported.
+
+==== Request Cache Settings
+
+The deprecated settings `index.cache.query.enable` and
+`indices.cache.query.size` have been removed and are replaced with
+`index.requests.cache.enable` and `indices.requests.cache.size` respectively.
+
+`indices.requests.cache.clean_interval has been replaced with
+`indices.cache.clean_interval and is no longer supported.
+
+==== Field Data Cache Settings
+
+The `indices.fielddata.cache.clean_interval` setting has been replaced with
+`indices.cache.clean_interval`.
+
+==== Allocation settings
+
+The `cluster.routing.allocation.concurrent_recoveries` setting has been
+replaced with `cluster.routing.allocation.node_concurrent_recoveries`.
+
+==== Similarity settings
+
+The 'default' similarity has been renamed to 'classic'.
+
+==== Indexing settings
+
+The `indices.memory.min_shard_index_buffer_size` and
+`indices.memory.max_shard_index_buffer_size` have been removed as
+Elasticsearch now allows any one shard to use  amount of heap as long as the
+total indexing buffer heap used across all shards is below the node's
+`indices.memory.index_buffer_size` (defaults to 10% of the JVM heap).
+
+==== Removed es.max-open-files
+
+Setting the system property es.max-open-files to true to get
+Elasticsearch to print the number of maximum open files for the
+Elasticsearch process has been removed. This same information can be
+obtained from the <<cluster-nodes-info>> API, and a warning is logged
+on startup if it is set too low.
+
+==== Removed es.netty.gathering
+
+Disabling Netty from using NIO gathering could be done via the escape
+hatch of setting the system property "es.netty.gathering" to "false".
+Time has proven enabling gathering by default is a non-issue and this
+non-documented setting has been removed.
+
+==== Removed es.useLinkedTransferQueue
+
+The system property `es.useLinkedTransferQueue` could be used to
+control the queue implementation used in the cluster service and the
+handling of ping responses during discovery. This was an undocumented
+setting and has been removed.
+
+==== Cache concurrency level settings removed
+
+Two cache concurrency level settings
+`indices.requests.cache.concurrency_level` and
+`indices.fielddata.cache.concurrency_level` because they no longer apply to
+the cache implementation used for the request cache and the field data cache.
+
+==== Using system properties to configure Elasticsearch
+
+Elasticsearch can be configured by setting system properties on the
+command line via `-Des.name.of.property=value.of.property`. This will be
+removed in a future version of Elasticsearch. Instead, use
+`-E es.name.of.setting=value.of.setting`. Note that in all cases the
+name of the setting must be prefixed with `es.`.
+
+==== Removed using double-dashes to configure Elasticsearch
+
+Elasticsearch could previously be configured on the command line by
+setting settings via `--name.of.setting value.of.setting`. This feature
+has been removed. Instead, use
+`-Ees.name.of.setting=value.of.setting`. Note that in all cases the
+name of the setting must be prefixed with `es.`.
+
+==== Discovery Settings
+
+The `discovery.zen.minimum_master_node` must bet set for nodes that are bound
+to a non-loopback network interface. We see those nodes as in "production" mode and
+thus require the setting.
+
+
diff --git a/docs/reference/modules.asciidoc b/docs/reference/modules.asciidoc
index 5ef8a41d3f5..b71d1224e7e 100644
--- a/docs/reference/modules.asciidoc
+++ b/docs/reference/modules.asciidoc
@@ -45,6 +45,10 @@ The modules in this section are:
 <<modules-node,Node client>>::
 
     A Java node client joins the cluster, but doesn't hold data or act as a master node.
+    
+<<modules-scripting-painless,Painless>>::
+
+    A built-in scripting language for Elasticsearch that's designed to be as secure as possible. 
 
 <<modules-plugins,Plugins>>::
 
@@ -53,7 +57,8 @@ The modules in this section are:
 <<modules-scripting,Scripting>>::
 
     Custom scripting available in Lucene Expressions, Groovy, Python, and
-    Javascript.
+    Javascript. You can also write scripts in the built-in scripting language, 
+    <<modules-scripting-painless, Painless>>.
 
 <<modules-snapshots,Snapshot/Restore>>::
 
@@ -89,6 +94,8 @@ include::modules/network.asciidoc[]
 
 include::modules/node.asciidoc[]
 
+include::modules/painless.asciidoc[]
+
 include::modules/plugins.asciidoc[]
 
 include::modules/scripting.asciidoc[]
diff --git a/docs/reference/modules/cluster/allocation_awareness.asciidoc b/docs/reference/modules/cluster/allocation_awareness.asciidoc
index ee3cbc17f5f..5735b52a1a8 100644
--- a/docs/reference/modules/cluster/allocation_awareness.asciidoc
+++ b/docs/reference/modules/cluster/allocation_awareness.asciidoc
@@ -21,7 +21,7 @@ attribute called `rack_id` -- we could use any attribute name.  For example:
 
 [source,sh]
 ----------------------
-./bin/elasticsearch --node.rack_id rack_one <1>
+./bin/elasticsearch -Ees.node.rack_id=rack_one <1>
 ----------------------
 <1> This setting could also be specified in the `elasticsearch.yml` config file.
 
diff --git a/docs/reference/modules/discovery/zen.asciidoc b/docs/reference/modules/discovery/zen.asciidoc
index 4c750fd0c19..59076fd3874 100644
--- a/docs/reference/modules/discovery/zen.asciidoc
+++ b/docs/reference/modules/discovery/zen.asciidoc
@@ -56,11 +56,9 @@ serves as a protection against (partial) network failures where node may unjustl
 think that the master has failed. In this case the node will simply hear from
 other nodes about the currently active master.
 
-If `discovery.zen.master_election.filter_client` is `true`, pings from client nodes (nodes where `node.client` is
-`true`, or both `node.data` and `node.master` are `false`) are ignored during master election; the default value is
-`true`. If `discovery.zen.master_election.filter_data` is `true`, pings from non-master-eligible data nodes (nodes
-where `node.data` is `true` and `node.master` is `false`) are ignored during master election; the default value is
-`false`. Pings from master-eligible nodes are always observed during master election.
+If `discovery.zen.master_election.ignore_non_master_pings` is `true`, pings from nodes that are not master
+eligible (nodes where `node.master` is `false`) are ignored during master election; the default value is
+`false`.
 
 Nodes can be excluded from becoming a master by setting `node.master` to
 `false`. Note, once a node is a client node (`node.client` set to
diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc
index 0117d193043..535b81f97c2 100644
--- a/docs/reference/modules/node.asciidoc
+++ b/docs/reference/modules/node.asciidoc
@@ -40,11 +40,20 @@ A tribe node, configured via the `tribe.*` settings, is a special type  of
 client node that can connect to multiple clusters and perform search and other
 operations across all connected clusters.
 
+<<ingest,Ingest node>>::
+
+A node that has `node.ingest` set to `true` (default). Ingest nodes are able
+to apply an <<pipeline,ingest pipeline>> to a document in order to transform
+and enrich the document before indexing. With a heavy ingest load, it makes
+sense to use dedicated ingest nodes and to mark the master and data nodes as
+`node.ingest: false`.
+
 By default a node is both a master-eligible node and a data node. This is very
 convenient for small clusters but, as the cluster grows, it becomes important
 to consider separating dedicated master-eligible nodes from dedicated data
 nodes.
 
+
 [NOTE]
 [[coordinating-node]]
 .Coordinating node
@@ -233,7 +242,7 @@ Like all node settings, it can also be specified on the command line as:
 
 [source,sh]
 -----------------------
-./bin/elasticsearch --path.data /var/elasticsearch/data
+./bin/elasticsearch -Ees.path.data=/var/elasticsearch/data
 -----------------------
 
 TIP: When using the `.zip` or `.tar.gz` distributions, the `path.data` setting
diff --git a/docs/reference/modules/painless.asciidoc b/docs/reference/modules/painless.asciidoc
new file mode 100644
index 00000000000..c52773b360e
--- /dev/null
+++ b/docs/reference/modules/painless.asciidoc
@@ -0,0 +1,728 @@
+[[modules-scripting-painless]]
+== Painless Scripting Language
+
+experimental[The Painless scripting language is new and is still marked as experimental. The syntax or API may be changed in the future in non-backwards compatible ways if required.]
+
+_Painless_ is a simple, secure scripting language built in to Elasticsearch as a module.
+It is designed specifically for use with Elasticsearch and can safely be used dynamically.
+
+A Painless script is essentially a single function. Painless does not provide support
+for defining multiple functions within a script. The Painless syntax is similar to
+http://groovy-lang.org/index.html[Groovy].
+
+You can use Painless anywhere a script can be used in Elasticsearch--simply set the `lang` parameter
+to `painless`.
+
+[[painless-features]]
+[float]
+== Painless Features
+
+* Control flow: `for` loops, `while` loops, `do/while` loops, `if/else`
+
+* Fully Typed: all available types/methods described in <<painless-api, Painless API>>
+
+* Arithmetic operators: multiplication `*`, division `/`, addition `+`, subtraction `-`, precedence `( )`
+
+* Comparison operators: less than `<`, less than or equal to `<=`, greater than `>`, greater than or equal to `>=`, equal to `==`, and not equal to `!=`, reference equals `===`, reference not equals `!==`
+
+* Boolean operators: not `!`, and `&&`, or `||`
+
+* Bitwise operators: shift left `<<`, shift right `>>`, unsigned shift `>>>`, and `&`, or `|`, xor `^`, not `~`
+
+* Shortcuts for list, map access using the dot `.` operator
+
+
+[[painless-examples]]
+[float]
+== Painless Examples
+
+To illustrate how Painless works, let's load some hockey stats into an Elasticsearch index:
+
+[source,sh]
+----------------------------------------------------------------
+DELETE /hockey-stats
+
+PUT /hockey-stats/player/_bulk
+{"index":{"_id":1}}
+{"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]}
+{"index":{"_id":2}}
+{"first":"sean","last":"monohan","goals":[7,54,26],"assists":[11,26,13],"gp":[26,82,82]}
+{"index":{"_id":3}}
+{"first":"jiri","last":"hudler","goals":[5,34,36],"assists":[11,62,42],"gp":[24,80,79]}
+{"index":{"_id":4}}
+{"first":"micheal","last":"frolik","goals":[4,6,15],"assists":[8,23,15],"gp":[26,82,82]}
+{"index":{"_id":5}}
+{"first":"sam","last":"bennett","goals":[5,0,0],"assists":[8,1,0],"gp":[26,1,0]}
+{"index":{"_id":6}}
+{"first":"dennis","last":"wideman","goals":[0,26,15],"assists":[11,30,24],"gp":[26,81,82]}
+{"index":{"_id":7}}
+{"first":"david","last":"jones","goals":[7,19,5],"assists":[3,17,4],"gp":[26,45,34]}
+{"index":{"_id":8}}
+{"first":"tj","last":"brodie","goals":[2,14,7],"assists":[8,42,30],"gp":[26,82,82]}
+{"index":{"_id":39}}
+{"first":"mark","last":"giordano","goals":[6,30,15],"assists":[3,30,24],"gp":[26,60,63]}
+{"index":{"_id":10}}
+{"first":"mikael","last":"backlund","goals":[3,15,13],"assists":[6,24,18],"gp":[26,82,82]}
+{"index":{"_id":11}}
+{"first":"joe","last":"colborne","goals":[3,18,13],"assists":[6,20,24],"gp":[26,67,82]}
+----------------------------------------------------------------
+// AUTOSENSE
+
+[float]
+=== Accessing Doc Values from Painless
+
+All Painless scripts take in a `Map<String,def>` of values called `input`.  Document values can be accessed through another `Map<String,def>` within the `input` variable.
+
+For example, the following script calculates a player's total goals. This example uses a strongly typed `int` and a `for` loop.
+
+[source,sh]
+----------------------------------------------------------------
+GET /hockey-stats/_search
+{
+  "query": {
+    "function_score": {
+      "script_score": {
+        "script": {
+          "lang": "painless",
+          "inline": "int total = 0; for (int i = 0; i < input.doc.goals.size(); ++i) { total += input.doc.goals[i]; } return total;"
+        }
+      }
+    }
+  }
+}
+----------------------------------------------------------------
+// AUTOSENSE
+
+Alternatively, you could do the same thing using a script field instead of a function score:
+
+[source,sh]
+----------------------------------------------------------------
+GET /hockey-stats/_search
+{
+  "query": {
+    "match_all": {}
+  },
+  "script_fields": {
+    "total_goals": {
+      "script": {
+        "lang": "painless",
+        "inline": "int total = 0; for (int i = 0; i < input.doc.goals.size(); ++i) { total += input.doc.goals[i]; } return total;"
+      }
+    }
+  }
+}
+----------------------------------------------------------------
+// AUTOSENSE
+
+You must always specify the index of the field value you want, even if there's only a single item in the field.
+All fields in Elasticsearch are multi-valued and Painless does not provide a `.value` shortcut. The following example uses a Painless script to sort the players by their combined first and last names. The names are accessed using
+`input.doc.first.0` and `input.doc.last.0`.
+
+[source,sh]
+----------------------------------------------------------------
+GET /hockey-stats/_search
+{
+  "query": {
+    "match_all": {}
+  },
+  "sort": {
+    "_script": {
+      "type": "string",
+      "order": "asc",
+      "script": {
+        "lang": "painless",
+        "inline": "input.doc.first.0 + \" \" + input.doc.last.0"
+      }
+    }
+  }
+}
+----------------------------------------------------------------
+// AUTOSENSE
+
+[float]
+=== Updating Fields with Painless
+
+You can also easily update fields. You access the original source for a field as `input.ctx._source.<field-name>`.
+
+First, let's look at the source data for a player by submitting the following request:
+
+[source,sh]
+----------------------------------------------------------------
+GET /hockey-stats/_search
+{
+  "fields": [
+    "_id",
+    "_source"
+  ],
+  "query": {
+    "term": {
+      "_id": 1
+    }
+  }
+}
+----------------------------------------------------------------
+// AUTOSENSE
+
+To change player 1's last name to `hockey`, simply set `input.ctx._source.last` to the new value:
+
+[source,sh]
+----------------------------------------------------------------
+POST /hockey-stats/player/1/_update
+{
+  "script": {
+    "lang": "painless",
+    "inline": "input.ctx._source.last = input.last",
+    "params": {
+      "last": "hockey"
+    }
+  }
+}
+----------------------------------------------------------------
+// AUTOSENSE
+
+You can also add fields to a document. For example, this script adds a new field that contains
+the player's nickname,  _hockey_.
+
+[source,sh]
+----------------------------------------------------------------
+POST /hockey-stats/player/1/_update
+{
+  "script": {
+    "lang": "painless",
+    "inline": "input.ctx._source.last = input.last input.ctx._source.nick = input.nick",
+    "params": {
+      "last": "gaudreau",
+      "nick": "hockey"
+    }
+  }
+}
+----------------------------------------------------------------
+// AUTOSENSE
+
+[float]
+=== Writing Type-Safe Scripts to Improve Performance
+
+If you explicitly specify types, the compiler doesn't have to perform type lookups at runtime, which can significantly
+improve performance. For example, the following script performs the same first name, last name sort we showed before,
+but it's fully type-safe.
+
+[source,sh]
+----------------------------------------------------------------
+GET /hockey-stats/_search
+{
+  "query": {
+    "match_all": {}
+  },
+  "script_fields": {
+    "full_name_dynamic": {
+      "script": {
+        "lang": "painless",
+        "inline": "def first = input.doc.first.0; def last = input.doc.last.0; return first + \" \" + last;"
+      }
+    },
+    "full_name_static": {
+      "script": {
+        "lang": "painless",
+        "inline": "String first = (String)((List)((Map)input.get(\"doc\")).get(\"first\")).get(0); String last = (String)((List)((Map)input.get(\"doc\")).get(\"last\")).get(0); return first + \" \" + last;"
+      }
+    }
+  }
+}
+----------------------------------------------------------------
+// AUTOSENSE
+
+[[painless-api]]
+[float]
+== Painless API
+
+The following types are available for use in the Painless language. Most types and methods map directly to their Java equivalents--for more information, see the corresponding https://docs.oracle.com/javase/8/docs/api/java/lang/package-summary.html[Javadoc].
+
+
+[float]
+=== Dynamic Types
+
+* `def` (This type can be used to represent any other type.)
+
+[float]
+=== Basic Types
+
+* `void`
+* `boolean`
+* `short`
+* `char`
+* `int`
+* `long`
+* `float`
+* `double`
+
+[float]
+=== Complex Types
+
+Non-static methods/members in superclasses are available to subclasses.
+Generic types with unspecified generic parameters are parameters of type `def`.
+
+-----
+ArithmeticException extends Exception
+   <init>()
+-----
+
+-----
+ArrayList extends List
+   <init>()
+-----
+
+-----
+ArrayList<Object> extends List<Object>
+   <init>()
+-----
+
+-----
+ArrayList<String> extends List<String>
+    <init>()
+-----
+
+-----
+Boolean extends Object
+   <init>(boolean)
+   static Boolean valueOf(boolean)
+   boolean booleanValue()
+-----
+
+-----
+Character extends Object
+    <init>(char)
+    static Character valueOf(char)
+    char charValue()
+    static char MIN_VALUE
+    static char MAX_VALUE
+-----
+
+-----
+CharSequence extends Object
+    char charAt(int)
+    int length()
+-----
+
+-----
+Collection extends Object
+    boolean add(def)
+    void clear()
+    boolean contains(def)
+    boolean isEmpty()
+    Iterator iterator()
+    boolean remove(def)
+    int size()
+-----
+
+-----
+Collection<Object> extends Object
+    boolean add(Object)
+    void clear()
+    boolean contains(Object)
+    boolean isEmpty()
+    Iterator iterator()
+    boolean remove(Object)
+    int size()
+-----
+
+-----
+Collection<String> extends Object
+    boolean add(String)
+    void clear()
+    boolean contains(String)
+    boolean isEmpty()
+    Iterator iterator()
+    boolean remove(String)
+    int size()
+-----
+
+-----
+Double extends Number
+    <init>(double)
+    static Double valueOf(double)
+    static double MIN_VALUE
+    static double MAX_VALUE
+-----
+
+-----
+Exception extends Object
+    String getMessage()
+-----
+
+-----
+Float extends Number
+    <init>(float)
+    static Float valueOf(float)
+    static float MIN_VALUE
+    static float MAX_VALUE
+-----
+
+-----
+HashMap extends Map
+    <init>()
+-----
+
+-----
+HashMap<Object,Object> extends Map<Object,Object>
+    <init>()
+-----
+
+-----
+HashMap<String,def> extends Map<String,def>
+    <init>()
+-----
+
+-----
+HashMap<String,Object> extends Map<String,Object>
+    <init>()
+-----
+
+-----
+IllegalArgument extends Exception
+    <init>()
+-----
+
+-----
+IllegalState extends Exception
+    <init>()
+-----
+
+-----
+Integer extends Number
+    <init>(int)
+    static Integer valueOf(int)
+    static int MIN_VALUE
+    static int MAX_VALUE
+-----
+
+-----
+Iterator extends Object
+    boolean hasNext()
+    def next()
+    void remove()
+-----
+
+-----
+Iterator<String> extends Object
+    boolean hasNext()
+    String next()
+    void remove()
+-----
+
+-----
+List extends Collection
+    def set(int, def)
+    def get(int)
+    def remove(int)
+-----
+
+-----
+List<Object> extends Collection
+    Object set(int, Object)
+    Object get(int)
+    Object remove(int)
+-----
+
+-----
+List<String> extends Collection
+    String set(int, String)
+    String get(int)
+    String remove(int)
+-----
+
+-----
+Long extends Number
+    <init>(long)
+    static Long valueOf(long)
+    static long MIN_VALUE
+    static long MAX_VALUE
+-----
+
+-----
+Map extends Object
+    def put (def, def)
+    def get (def)
+    def remove (def)
+    boolean isEmpty()
+    int size()
+    boolean containsKey(def)
+    boolean containsValue(def)
+    Set keySet()
+    Collection values()
+-----
+
+-----
+Map<Object,Object> extends Object
+    Object put (Object, Object)
+    Object get (Object)
+    Object remove (Object)
+    boolean isEmpty()
+    int size()
+    boolean containsKey(Object)
+    boolean containsValue(Object)
+    Set keySet()
+    Collection values()
+-----
+
+-----
+Map<String,def> extends Object
+    def put (String, def)
+    def get (String)
+    def remove (String)
+    boolean isEmpty()
+    int size()
+    boolean containsKey(String)
+    boolean containsValue(def)
+    Set<String> keySet()
+    Collection values()
+-----
+
+-----
+Map<String,Object> extends Object
+    Object put (String, Object)
+    Object get (String)
+    Object remove (String)
+    boolean isEmpty()
+    int size()
+    boolean containsKey(String)
+    boolean containsValue(Object)
+    Set<String> keySet()
+    Collection values()
+-----
+
+-----
+Number extends Object
+    short shortValue()
+    short shortValue()
+    int intValue()
+    long longValue()
+    float floatValue()
+    double doubleValue()
+-----
+
+-----
+Object
+    String toString()
+    boolean equals(Object)
+    int hashCode()
+-----
+
+-----
+Set extends Collection
+-----
+
+-----
+Set<Object> extends Collection<Object>
+-----
+
+-----
+Set<String> extends Collection<String>
+-----
+
+-----
+Short extends Number
+    <init>(short)
+    static Short valueOf(short)
+    static short MIN_VALUE
+    static short MAX_VALUE
+-----
+
+-----
+String extends CharSequence
+    <init>(String)
+    int codePointAt(int)
+    int compareTo(String)
+    String concat(String)
+    boolean endsWith(String)
+    int indexOf(String, int)
+    boolean isEmpty()
+    String replace(CharSequence, CharSequence)
+    boolean startsWith(String)
+    String substring(int, int)
+    char[] toCharArray()
+    String trim()
+-----
+
+-----
+NumberFormatException extends Exception
+    <init>()
+-----
+
+-----
+Void extends Object
+-----
+
+[float]
+==== Utility Classes
+
+-----
+Math
+   static double abs(double)
+   static float fabs(float)
+   static long labs(long)
+   static int iabs(int)
+   static double acos(double)
+   static double asin(double)
+   static double atan(double)
+   static double atan2(double)
+   static double cbrt(double)
+   static double ceil(double)
+   static double cos(double)
+   static double cosh(double)
+   static double exp(double)
+   static double expm1(double)
+   static double floor(double)
+   static double hypt(double, double)
+   static double abs(double)
+   static double log(double)
+   static double log10(double)
+   static double log1p(double)
+   static double max(double, double)
+   static float fmax(float, float)
+   static long lmax(long, long)
+   static int imax(int, int)
+   static double min(double, double)
+   static float fmin(float, float)
+   static long lmin(long, long)
+   static int imin(int, int)
+   static double pow(double, double)
+   static double random()
+   static double rint(double)
+   static long round(double)
+   static double sin(double)
+   static double sinh(double)
+   static double sqrt(double)
+   static double tan(double)
+   static double tanh(double)
+   static double toDegrees(double)
+   static double toRadians(double)
+-----
+
+-----
+Utility
+   static boolean NumberToboolean(Number)
+   static char NumberTochar(Number)
+   static Boolean NumberToBoolean(Number)
+   static Short NumberToShort(Number)
+   static Character NumberToCharacter(Number)
+   static Integer NumberToInteger(Number)
+   static Long NumberToLong(Number)
+   static Float NumberToFloat(Number)
+   static Double NumberToDouble(Number)
+   static byte booleanTobyte(boolean)
+   static short booleanToshort(boolean)
+   static char booleanTochar(boolean)
+   static int booleanToint(boolean)
+   static long booleanTolong(boolean)
+   static float booleanTofloat(boolean)
+   static double booleanTodouble(boolean)
+   static Integer booleanToInteger(boolean)
+   static byte BooleanTobyte(Boolean)
+   static short BooleanToshort(Boolean)
+   static char BooleanTochar(Boolean)
+   static int BooleanToint(Boolean)
+   static long BooleanTolong(Boolean)
+   static float BooleanTofloat(Boolean)
+   static double BooleanTodouble(Boolean)
+   static Byte BooleanToByte(Boolean)
+   static Short BooleanToShort(Boolean)
+   static Character BooleanToCharacter(Boolean)
+   static Integer BooleanToInteger(Boolean)
+   static Long BooleanToLong(Boolean)
+   static Float BooleanToFloat(Boolean)
+   static Double BooleanToDouble(Boolean)
+   static boolean byteToboolean(byte)
+   static Short byteToShort(byte)
+   static Character byteToCharacter(byte)
+   static Integer byteToInteger(byte)
+   static Long byteToLong(byte)
+   static Float byteToFloat(byte)
+   static Double byteToDouble(byte)
+   static boolean ByteToboolean(Byte)
+   static char ByteTochar(Byte)
+   static boolean shortToboolean(short)
+   static Byte shortToByte(short)
+   static Character shortToCharacter(short)
+   static Integer shortToInteger(short)
+   static Long shortToLong(short)
+   static Float shortToFloat(short)
+   static Double shortToDouble(short)
+   static boolean ShortToboolean(Short)
+   static char ShortTochar(Short)
+   static boolean charToboolean(char)
+   static Byte charToByte(char)
+   static Short charToShort(char)
+   static Integer charToInteger(char)
+   static Long charToLong(char)
+   static Float charToFloat(char)
+   static Double charToDouble(char)
+   static boolean CharacterToboolean(Character)
+   static byte CharacterTobyte(Character)
+   static short CharacterToshort(Character)
+   static int CharacterToint(Character)
+   static long CharacterTolong(Character)
+   static float CharacterTofloat(Character)
+   static double CharacterTodouble(Character)
+   static Boolean CharacterToBoolean(Character)
+   static Byte CharacterToByte(Character)
+   static Short CharacterToShort(Character)
+   static Integer CharacterToInteger(Character)
+   static Long CharacterToLong(Character)
+   static Float CharacterToFloat(Character)
+   static Double CharacterToDouble(Character)
+   static boolean intToboolean(int)
+   static Byte intToByte(int)
+   static Short intToShort(int)
+   static Character intToCharacter(int)
+   static Long intToLong(int)
+   static Float intToFloat(int)
+   static Double intToDouble(int)
+   static boolean IntegerToboolean(Integer)
+   static char IntegerTochar(Integer)
+   static boolean longToboolean(long)
+   static Byte longToByte(long)
+   static Short longToShort(long)
+   static Character longToCharacter(long)
+   static Integer longToInteger(long)
+   static Float longToFloat(long)
+   static Double longToDouble(long)
+   static boolean LongToboolean(Long)
+   static char LongTochar(Long)
+   static boolean floatToboolean(float)
+   static Byte floatToByte(float)
+   static Short floatToShort(float)
+   static Character floatToCharacter(float)
+   static Integer floatToInteger(float)
+   static Long floatToLong(float)
+   static Double floatToDouble(float)
+   static boolean FloatToboolean(Float)
+   static char FloatTochar(Float)
+   static boolean doubleToboolean(double)
+   static Byte doubleToByte(double)
+   static Short doubleToShort(double)
+   static Character doubleToCharacter(double)
+   static Integer doubleToInteger(double)
+   static Long doubleToLong(double)
+   static Float doubleToFloat(double)
+   static boolean DoubleToboolean(Double)
+   static char DoubleTochar(Double)
+-----
+
+-----
+Def
+    static boolean defToboolean(def)
+    static byte defTobyte(def)
+    static short defToshort(def)
+    static char defTochar(def)
+    static int defToint(def)
+    static long defTolong(def)
+    static float defTofloat(def)
+    static double defTodouble(def)
+    static Boolean defToBoolean(def)
+    static Byte defToByte(def)
+    static Character defToCharacter(def)
+    static Integer defToInteger(def)
+    static Long defToLong(def)
+    static Float defToFloat(def)
+    static Double defToDouble(def)
+-----
\ No newline at end of file
diff --git a/docs/reference/modules/scripting.asciidoc b/docs/reference/modules/scripting.asciidoc
index f4374a0f9b3..114740306ee 100644
--- a/docs/reference/modules/scripting.asciidoc
+++ b/docs/reference/modules/scripting.asciidoc
@@ -3,4 +3,3 @@ include::scripting/scripting.asciidoc[]
 include::scripting/advanced-scripting.asciidoc[]
 
 include::scripting/security.asciidoc[]
-
diff --git a/docs/reference/modules/scripting/scripting.asciidoc b/docs/reference/modules/scripting/scripting.asciidoc
index 4f9d84f34f8..12524407900 100644
--- a/docs/reference/modules/scripting/scripting.asciidoc
+++ b/docs/reference/modules/scripting/scripting.asciidoc
@@ -1,27 +1,48 @@
 [[modules-scripting]]
 == Scripting
 
-The scripting module allows to use scripts in order to evaluate custom
-expressions. For example, scripts can be used to return "script fields"
-as part of a search request, or can be used to evaluate a custom score
-for a query and so on.
+The scripting module enables you to use scripts to evaluate custom
+expressions. For example, you could use a script to return "script fields"
+as part of a search request or evaluate a custom score for a query.
 
-The scripting module uses by default http://groovy-lang.org/[groovy]
-(previously http://mvel.codehaus.org/[mvel] in 1.3.x and earlier) as the
-scripting language with some extensions. Groovy is used since it is extremely
-fast and very simple to use.
+TIP: Elasticsearch now has a built-in scripting language called _Painless_
+that provides a more secure alternative for implementing
+scripts for Elasticsearch. We encourage you to try it out--
+for more information, see <<modules-scripting-painless, Painless Scripting Language>>.
+
+The default scripting language is http://groovy-lang.org/[groovy] 
+(http://mvel.codehaus.org/[mvel] was the default in 1.3.x and earlier). 
+
+Additional `lang` plugins enable you to run scripts written in other languages. 
+Everywhere a script can be used, you can include a `lang` parameter
+to specify the language of the script. Plugins are available for following languages:
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Language   |Sandboxed |Required plugin
+|groovy     |no        |built-in
+|expression |yes       |built-in
+|mustache   |yes       |built-in
+/painless   /yes       /built-in (module)
+|javascript |no        |{plugins}/lang-javascript.html[elasticsearch-lang-javascript]
+|python     |no        |{plugins}/lang-python.html[elasticsearch-lang-python]
+|=======================================================================
 
 .Groovy dynamic scripting off by default from v1.4.3
 [IMPORTANT]
 ===================================================
 
-Groovy dynamic scripting is off by default, preventing dynamic Groovy scripts
-from being accepted as part of a request or retrieved from the special
-`.scripts` index. You will still be able to use Groovy scripts stored in files
-in the `config/scripts/` directory on every node.
+Groovy dynamic scripting is off by default. This prevents Groovy scripts
+from being accepted as part of a request or retrieved from the 
+`.scripts` index. You can still use Groovy file scripts stored in 
+the `config/scripts/` directory on every node.
 
-To convert an inline script to a file, take this simple script
-as an example:
+To convert an inline script to a file-based script, save the contents
+of the `inline` field to a file with the `.groovy` extension and
+store it in the `config/scripts` directory on every data node in your
+cluster. 
+
+For example, if you have the following inline script:
 
 [source,js]
 -----------------------------------
@@ -38,15 +59,9 @@ GET /_search
 }
 -----------------------------------
 
-Save the contents of the `inline` field as a file called `config/scripts/my_script.groovy`
-on every data node in the cluster:
+Save `1 + my_var` in a file called `config/scripts/my_script.groovy`.
 
-[source,js]
------------------------------------
-1 + my_var
------------------------------------
-
-Now you can access the script by file name (without the extension):
+To use the script in a request, specify its name (without the `.groovy` extension) in the `file` field:
 
 [source,js]
 -----------------------------------
@@ -67,21 +82,8 @@ GET /_search
 
 ===================================================
 
-
-Additional `lang` plugins are provided to allow to execute scripts in
-different languages. All places where a script can be used, a `lang` parameter
-can be provided to define the language of the script. The following are the
-supported scripting languages:
-
-[cols="<,<,<",options="header",]
-|=======================================================================
-|Language   |Sandboxed |Required plugin
-|groovy     |no        |built-in
-|expression |yes       |built-in
-|mustache   |yes       |built-in
-|javascript |no        |{plugins}/lang-javascript.html[elasticsearch-lang-javascript]
-|python     |no        |{plugins}/lang-python.html[elasticsearch-lang-python]
-|=======================================================================
+[float]
+=== File-based Scripts
 
 To increase security, Elasticsearch does not allow you to specify scripts for
 non-sandboxed languages with a request. Instead, scripts must be placed in the
@@ -219,8 +221,6 @@ Indexed scripts can be deleted by:
 curl -XDELETE localhost:9200/_scripts/groovy/indexedCalculateScore
 -----------------------------------
 
-
-
 [float]
 [[enable-dynamic-scripting]]
 === Enabling dynamic scripting
diff --git a/docs/reference/modules/scripting/security.asciidoc b/docs/reference/modules/scripting/security.asciidoc
index af193b35103..e84289bf1d9 100644
--- a/docs/reference/modules/scripting/security.asciidoc
+++ b/docs/reference/modules/scripting/security.asciidoc
@@ -100,7 +100,13 @@ Security Policy either:
 
 * system wide: `$JAVA_HOME/lib/security/java.policy`,
 * for just the `elasticsearch` user: `/home/elasticsearch/.java.policy`, or
-* from a file specified on the command line: `-Djava.security.policy=someURL`
+* from a file specified in the `JAVA_OPTS` environment variable with `-Djava.security.policy=someURL`:
++
+[source,js]
+---------------------------------
+export JAVA_OPTS="${JAVA_OPTS} -Djava.security.policy=file:///path/to/my.policy`
+./bin/elasticsearch
+---------------------------------
 
 Permissions may be granted at the class, package, or global level.  For instance:
 
diff --git a/docs/reference/modules/threadpool.asciidoc b/docs/reference/modules/threadpool.asciidoc
index bfd5474183c..199b6a9b88c 100644
--- a/docs/reference/modules/threadpool.asciidoc
+++ b/docs/reference/modules/threadpool.asciidoc
@@ -19,15 +19,10 @@ There are several thread pools, but the important ones include:
     queue_size of `200`.
 
 `search`::
-    For count/search operations. Thread pool type is `fixed`
+    For count/search/suggest operations. Thread pool type is `fixed`
     with a size of `int((# of available_processors * 3) / 2) + 1`,
     queue_size of `1000`.
 
-`suggest`::
-    For suggest operations. Thread pool type is `fixed`
-    with a size of `# of available processors`,
-    queue_size of `1000`.
-
 `get`::
     For get operations. Thread pool type is `fixed`
     with a size of `# of available processors`,
diff --git a/docs/reference/modules/tribe.asciidoc b/docs/reference/modules/tribe.asciidoc
index bdae1816fba..6e1dbb75e40 100644
--- a/docs/reference/modules/tribe.asciidoc
+++ b/docs/reference/modules/tribe.asciidoc
@@ -7,7 +7,8 @@ multiple clusters.
 The tribe node works by retrieving the cluster state from all connected
 clusters and merging them into a global cluster state. With this information
 at hand, it is able to perform read and write operations against the nodes in
-all clusters as if they were local.
+all clusters as if they were local. Note that a tribe node needs to be able
+to connect to each single node in every configured cluster.
 
 The `elasticsearch.yml` config file for a tribe node just needs to list the
 clusters that should be joined, for instance:
diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc
index f8e08aaada9..dc986bf14d9 100644
--- a/docs/reference/query-dsl/bool-query.asciidoc
+++ b/docs/reference/query-dsl/bool-query.asciidoc
@@ -141,3 +141,10 @@ GET _search
 }
 ---------------------------------
 // AUTOSENSE
+
+==== Using named queries to see which clauses matched
+
+If you need to know which of the clauses in the bool query matched the documents
+returned from the query, you can use
+<<search-request-named-queries-and-filters,named queries>> to assign a name to
+each clause.
diff --git a/docs/reference/query-dsl/exists-query.asciidoc b/docs/reference/query-dsl/exists-query.asciidoc
index 404dce4a4ae..b484d47f4b6 100644
--- a/docs/reference/query-dsl/exists-query.asciidoc
+++ b/docs/reference/query-dsl/exists-query.asciidoc
@@ -47,7 +47,7 @@ instance, if the `user` field were mapped as follows:
 [source,js]
 --------------------------------------------------
   "user": {
-    "type": "string",
+    "type": "text",
     "null_value": "_null_"
   }
 --------------------------------------------------
diff --git a/docs/reference/query-dsl/function-score-query.asciidoc b/docs/reference/query-dsl/function-score-query.asciidoc
index 39fdae80242..1e736efa11d 100644
--- a/docs/reference/query-dsl/function-score-query.asciidoc
+++ b/docs/reference/query-dsl/function-score-query.asciidoc
@@ -215,6 +215,22 @@ There are a number of options for the `field_value_factor` function:
     `log1p`, `log2p`, `ln`, `ln1p`, `ln2p`, `square`, `sqrt`, or `reciprocal`.
     Defaults to `none`.
 
+[cols="<,<",options="header",]
+|=======================================================================
+| Modifier | Meaning
+
+| `none` | Do not apply any multiplier to the field value
+| `log` | Take the https://en.wikipedia.org/wiki/Logarithm[logarithm] of the field value
+| `log1p` | Add 1 to the field value and take the logarithm
+| `log2p` | Add 2 to the field value and take the logarithm
+| `ln` | Take the https://en.wikipedia.org/wiki/Natural_logarithm[natural logarithm] of the field value
+| `ln1p` | Add 1 to the field value and take the natural logarithm
+| `ln2p` | Add 2 to the field value and take the natural logarithm
+| `square` | Square the field value (multiply it by itself)
+| `sqrt` | Take the https://en.wikipedia.org/wiki/Square_root[square root] of the field value
+| `reciprocal` | https://en.wikipedia.org/wiki/Multiplicative_inverse[Reciprocate] the field value, same as `1/x` where `x` is the field's value
+|=======================================================================
+
 `missing`::
 
     Value used if the document doesn't have that field. The modifier
diff --git a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
index c52bcb93e7d..b2103e5772f 100644
--- a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
+++ b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
@@ -52,9 +52,6 @@ Then the following simple query can be executed with a
 |Option |Description
 |`_name` |Optional name field to identify the filter
 
-|`coerce` |Set to `true` to normalize longitude and latitude values to a
-standard -180:180 / -90:90 coordinate system. (default is `false`).
-
 |`ignore_malformed` |Set to `true` to
 accept geo points with invalid latitude or longitude (default is `false`).
 
@@ -188,10 +185,10 @@ values separately.
         "filter" : {
             "geo_bounding_box" : {
                 "pin.location" : {
-                    "top" : -74.1,
-                    "left" : 40.73,
-                    "bottom" : -71.12,
-                    "right" : 40.01
+                    "top" : 40.73,
+                    "left" : -74.1,
+                    "bottom" : 40.01,
+                    "right" : -71.12
                 }
             }
         }
diff --git a/docs/reference/query-dsl/geo-distance-query.asciidoc b/docs/reference/query-dsl/geo-distance-query.asciidoc
index c5b6029dc2f..7ea380bdad2 100644
--- a/docs/reference/query-dsl/geo-distance-query.asciidoc
+++ b/docs/reference/query-dsl/geo-distance-query.asciidoc
@@ -162,11 +162,6 @@ The following are options allowed on the filter:
 
     Optional name field to identify the query
 
-`coerce`::
-
-    Set to `true` to normalize longitude and latitude values to a standard -180:180 / -90:90
-    coordinate system. (default is `false`).
-
 `ignore_malformed`::
 
     Set to `true` to accept geo points with invalid latitude or
diff --git a/docs/reference/query-dsl/geo-polygon-query.asciidoc b/docs/reference/query-dsl/geo-polygon-query.asciidoc
index 306b2dd2d84..269aeed09ca 100644
--- a/docs/reference/query-dsl/geo-polygon-query.asciidoc
+++ b/docs/reference/query-dsl/geo-polygon-query.asciidoc
@@ -34,9 +34,6 @@ points. Here is an example:
 |Option |Description
 |`_name` |Optional name field to identify the filter
 
-|`coerce` |Set to `true` to normalize longitude and latitude values to a
-standard -180:180 / -90:90 coordinate system. (default is `false`).
-
 |`ignore_malformed` |Set to `true` to accept geo points with invalid latitude or
 longitude (default is `false`).
 |=======================================================================
diff --git a/docs/reference/query-dsl/has-child-query.asciidoc b/docs/reference/query-dsl/has-child-query.asciidoc
index 01c3c35db54..b98646148f8 100644
--- a/docs/reference/query-dsl/has-child-query.asciidoc
+++ b/docs/reference/query-dsl/has-child-query.asciidoc
@@ -35,7 +35,7 @@ inside the `has_child` query:
 {
     "has_child" : {
         "type" : "blog_tag",
-        "score_mode" : "sum",
+        "score_mode" : "min",
         "query" : {
             "term" : {
                 "tag" : "something"
@@ -57,7 +57,7 @@ a match:
 {
     "has_child" : {
         "type" : "blog_tag",
-        "score_mode" : "sum",
+        "score_mode" : "min",
         "min_children": 2, <1>
         "max_children": 10, <1>
         "query" : {
diff --git a/docs/reference/query-dsl/mlt-query.asciidoc b/docs/reference/query-dsl/mlt-query.asciidoc
index ee4b695c2ff..d2d521f9492 100644
--- a/docs/reference/query-dsl/mlt-query.asciidoc
+++ b/docs/reference/query-dsl/mlt-query.asciidoc
@@ -73,7 +73,6 @@ present in the index, the syntax is similar to <<docs-termvectors-artificial-doc
                 },
                 "tweet": "You got no idea what I'd... what I'd give to be invisible."
               }
-            }
         },
         {
             "_index" : "marvel",
@@ -117,18 +116,18 @@ curl -s -XPUT 'http://localhost:9200/imdb/' -d '{
     "movies": {
       "properties": {
         "title": {
-          "type": "string",
+          "type": "text",
           "term_vector": "yes"
          },
          "description": {
-          "type": "string"
+          "type": "text"
         },
         "tags": {
-          "type": "string",
+          "type": "text",
           "fields" : {
             "raw": {
-              "type" : "string",
-              "index" : "not_analyzed",
+              "type" : "text",
+              "analyzer": "keyword",
               "term_vector" : "yes"
             }
           }
diff --git a/docs/reference/query-dsl/percolator-query.asciidoc b/docs/reference/query-dsl/percolator-query.asciidoc
new file mode 100644
index 00000000000..f2b1e2b9805
--- /dev/null
+++ b/docs/reference/query-dsl/percolator-query.asciidoc
@@ -0,0 +1,424 @@
+[[query-dsl-percolator-query]]
+=== Percolator Query
+
+Traditionally you design documents based on your data, store them into an index, and then define queries via the search API
+in order to retrieve these documents. The percolator works in the opposite direction. First you store queries into an
+index and then you use the `percolator` query to search for the queries which match a specified document (or documents).
+
+The reason that queries can be stored comes from the fact that in Elasticsearch both documents and queries are defined in
+JSON. This allows you to embed queries into documents via the index API. Elasticsearch can extract the query from a
+document and make it available for search via the `percolator` query. Since documents are also defined as JSON,
+you can define a document in the `percolator` query.
+
+[IMPORTANT]
+=====================================
+
+Fields referred to in a percolator query must *already* exist in the mapping
+associated with the index used for percolation. In order to make sure these fields exist,
+add or update a mapping via the <<indices-create-index,create index>> or <<indices-put-mapping,put mapping>> APIs.
+
+=====================================
+
+[float]
+=== Sample Usage
+
+Create an index with a mapping for the field `message`:
+
+[source,js]
+--------------------------------------------------
+curl -XPUT 'localhost:9200/my-index' -d '{
+  "mappings": {
+    "my-type": {
+      "properties": {
+        "message": {
+          "type": "string"
+        }
+      }
+    }
+  }
+}'
+--------------------------------------------------
+
+Register a query in the percolator:
+
+[source,js]
+--------------------------------------------------
+curl -XPUT 'localhost:9200/my-index/.percolator/1' -d '{
+    "query" : {
+        "match" : {
+            "message" : "bonsai tree"
+        }
+    }
+}'
+--------------------------------------------------
+
+Match a document to the registered percolator queries:
+
+[source,js]
+--------------------------------------------------
+curl -XGET 'localhost:9200/my-index/_search' -d '{
+    "query" : {
+        "percolator" : {
+            "document_type" : "my-type",
+            "document" : {
+                "message" : "A new bonsai tree in the office"
+            }
+        }
+    }
+}'
+--------------------------------------------------
+
+The above request will yield the following response:
+
+[source,js]
+--------------------------------------------------
+{
+  "took": 5,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "failed": 0
+  },
+  "hits": {
+    "total": 1,
+    "max_score": 0,
+    "hits": [
+      { <1>
+        "_index": "my-index",
+        "_type": ".percolator",
+        "_id": "1",
+        "_score": 0,
+        "_source": {
+          "query": {
+            "match": {
+              "message": "bonsai tree"
+            }
+          }
+        }
+      }
+    ]
+  }
+}
+--------------------------------------------------
+
+<1> The percolate query with id `1` matches our document.
+
+[float]
+=== Indexing Percolator Queries
+
+Percolate queries are stored as documents in a specific format and in an arbitrary index under a reserved type with the
+name `.percolator`. The query itself is placed as is in a JSON object under the top level field `query`.
+
+[source,js]
+--------------------------------------------------
+{
+    "query" : {
+		"match" : {
+			"field" : "value"
+		}
+	}
+}
+--------------------------------------------------
+
+Since this is just an ordinary document, any field can be added to this document. This can be useful later on to only
+percolate documents by specific queries.
+
+[source,js]
+--------------------------------------------------
+{
+	"query" : {
+		"match" : {
+			"field" : "value"
+		}
+	},
+	"priority" : "high"
+}
+--------------------------------------------------
+
+Just as with any other type, the `.percolator` type has a mapping, which you can configure via the mappings APIs.
+The default percolate mapping doesn't index the query field, only stores it.
+
+Because `.percolate` is a type it also has a mapping. By default the following mapping is active:
+
+[source,js]
+--------------------------------------------------
+{
+	".percolator" : {
+		"properties" : {
+			"query" : {
+				"type" : "percolator"
+			}
+		}
+	}
+}
+--------------------------------------------------
+
+If needed, this mapping can be modified with the update mapping API.
+
+In order to un-register a percolate query the delete API can be used. So if the previous added query needs to be deleted
+the following delete requests needs to be executed:
+
+[source,js]
+--------------------------------------------------
+curl -XDELETE localhost:9200/my-index/.percolator/1
+--------------------------------------------------
+
+[float]
+==== Parameters
+
+The following parameters are required when percolating a document:
+
+[horizontal]
+`document_type`:: The type / mapping of the document being percolated. This is parameter is always required.
+`document`:: The source of the document being percolated.
+
+Instead of specifying a the source of the document being percolated, the source can also be retrieved from an already
+stored document. The `percolator` query will then internally execute a get request to fetch that document.
+
+In that case the `document` parameter can be substituted with the following parameters:
+
+[horizontal]
+`index`:: The index the document resides in. This is a required parameter.
+`type`:: The type of the document to fetch. This is a required parameter.
+`id`:: The id of the document to fetch. This is a required parameter.
+`routing`:: Optionally, routing to be used to fetch document to percolate.
+`preference`:: Optionally, preference to be used to fetch document to percolate.
+`version`:: Optionally, the expected version of the document to be fetched.
+
+[float]
+==== Dedicated Percolator Index
+
+Percolate queries can be added to any index. Instead of adding percolate queries to the index the data resides in,
+these queries can also be added to a dedicated index. The advantage of this is that this dedicated percolator index
+can have its own index settings (For example the number of primary and replica shards). If you choose to have a dedicated
+percolate index, you need to make sure that the mappings from the normal index are also available on the percolate index.
+Otherwise percolate queries can be parsed incorrectly.
+
+[float]
+==== Percolating an Existing Document
+
+In order to percolate a newly indexed document, the `percolator` query can be used. Based on the response
+from an index request, the `_id` and other meta information can be used to immediately percolate the newly added
+document.
+
+[float]
+===== Example
+
+Based on the previous example.
+
+Index the document we want to percolate:
+
+[source,js]
+--------------------------------------------------
+curl -XPUT "http://localhost:9200/my-index/message/1" -d'
+{
+  "message" : "A new bonsai tree in the office"
+}'
+--------------------------------------------------
+
+Index response:
+
+[source,js]
+--------------------------------------------------
+{
+  "_index": "my-index",
+  "_type": "message",
+  "_id": "1",
+  "_version": 1,
+  "_shards": {
+    "total": 2,
+    "successful": 1,
+    "failed": 0
+  },
+  "created": true
+}
+--------------------------------------------------
+
+Percolating an existing document, using the index response as basis to build to new search request:
+
+[source,js]
+--------------------------------------------------
+curl -XGET "http://localhost:9200/my-index/_search" -d'
+{
+    "query" : {
+        "percolator" : {
+            "document_type" : "my-type",
+            "index" : "my-index",
+            "type" : "message",
+            "id" : "1",
+            "version" : 1 <1>
+        }
+    }
+}'
+--------------------------------------------------
+
+<1> The version is optional, but useful in certain cases. We can then ensure that we are try to percolate
+the document we just have indexed. A change may be made after we have indexed, and if that is the
+case the then the search request would fail with a version conflict error.
+
+The search response returned is identical as in the previous example.
+
+[float]
+==== Percolator and highlighting
+
+The percolator query is handled in a special way when it comes to highlighting. The percolator queries hits are used
+to highlight the document that is provided in the `percolator` query. Whereas with regular highlighting the query in
+the search request is used to highlight the hits.
+
+[float]
+===== Example
+
+This example is based on the mapping of the first example.
+
+Add a percolator query:
+
+[source,js]
+--------------------------------------------------
+curl -XPUT "http://localhost:9200/my-index/.percolator/1" -d'
+{
+    "query" : {
+        "match" : {
+            "message" : "brown fox"
+        }
+    }
+}'
+--------------------------------------------------
+
+Add another percolator query:
+
+[source,js]
+--------------------------------------------------
+curl -XPUT "http://localhost:9200/my-index/.percolator/2" -d'
+{
+    "query" : {
+        "match" : {
+            "message" : "lazy dog"
+        }
+    }
+}'
+--------------------------------------------------
+
+Execute a search request with `percolator` and highlighting enabled:
+
+[source,js]
+--------------------------------------------------
+curl -XGET "http://localhost:9200/my-index/_search" -d'
+{
+    "query" : {
+        "percolator" : {
+            "document_type" : "my-type",
+            "document" : {
+                "message" : "The quick brown fox jumps over the lazy dog"
+            }
+        }
+    },
+    "highlight": {
+      "fields": {
+        "message": {}
+      }
+    }
+}'
+--------------------------------------------------
+
+This will yield the following response.
+
+[source,js]
+--------------------------------------------------
+{
+  "took": 14,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "failed": 0
+  },
+  "hits": {
+    "total": 2,
+    "max_score": 0,
+    "hits": [
+      {
+        "_index": "my-index",
+        "_type": ".percolator",
+        "_id": "2",
+        "_score": 0,
+        "_source": {
+          "query": {
+            "match": {
+              "message": "lazy dog"
+            }
+          }
+        },
+        "highlight": {
+          "message": [
+            "The quick brown fox jumps over the <em>lazy</em> <em>dog</em>" <1>
+          ]
+        }
+      },
+      {
+        "_index": "my-index",
+        "_type": ".percolator",
+        "_id": "1",
+        "_score": 0,
+        "_source": {
+          "query": {
+            "match": {
+              "message": "brown fox"
+            }
+          }
+        },
+        "highlight": {
+          "message": [
+            "The quick <em>brown</em> <em>fox</em> jumps over the lazy dog" <1>
+          ]
+        }
+      }
+    ]
+  }
+}
+--------------------------------------------------
+
+<1> Instead of the query in the search request highlighting the percolator hits, the percolator queries are highlighting
+    the document defined in the `percolator` query.
+
+[float]
+==== How it Works Under the Hood
+
+When indexing a document that contains a query in an index and the `.percolator` type, the query part of the documents gets
+parsed into a Lucene query and is kept in memory until that percolator document is removed or the index containing the
+`.percolator` type gets removed. So, all the active percolator queries are kept in memory.
+
+At search time, the document specified in the request gets parsed into a Lucene document and is stored in a in-memory
+Lucene index. This in-memory index can just hold this one document and it is optimized for that. Then all the queries
+that are registered to the index that the searh request is targeted for, are going to be executed on this single document
+in-memory index. This happens on each shard the search request needs to execute.
+
+By using `routing` or additional queries the amount of percolator queries that need to be executed can be reduced and thus
+the time the search API needs to run can be decreased.
+
+[float]
+==== Important Notes
+
+Because the percolator query is processing one document at a time, it doesn't support queries and filters that run
+against child documents such as `has_child` and `has_parent`.
+
+The percolator doesn't work with queries like `template` and `geo_shape` queries when these queries fetch documents
+to substitute parts of the query. The reason is that the percolator stores the query terms during indexing in order to
+speedup percolating in certain cases and this doesn't work if part of the query is defined in another document.
+There is no way to know for the percolator to know if an external document has changed and even if this was the case the
+percolator query has to be reindexed.
+
+The `wildcard` and `regexp` query natively use a lot of memory and because the percolator keeps the queries into memory
+this can easily take up the available memory in the heap space. If possible try to use a `prefix` query or ngramming to
+achieve the same result (with way less memory being used).
+
+[float]
+==== Forcing Unmapped Fields to be Handled as Strings
+
+In certain cases it is unknown what kind of percolator queries do get registered, and if no field mapping exists for fields
+that are referred by percolator queries then adding a percolator query fails. This means the mapping needs to be updated
+to have the field with the appropriate settings, and then the percolator query can be added. But sometimes it is sufficient
+if all unmapped fields are handled as if these were default string fields. In those cases one can configure the
+`index.percolator.map_unmapped_fields_as_string` setting to `true` (default to `false`) and then if a field referred in
+a percolator query does not exist, it will be handled as a default string field so that adding the percolator query doesn't
+fail.
\ No newline at end of file
diff --git a/docs/reference/query-dsl/regexp-syntax.asciidoc b/docs/reference/query-dsl/regexp-syntax.asciidoc
index e57d0e1c779..68ca5912458 100644
--- a/docs/reference/query-dsl/regexp-syntax.asciidoc
+++ b/docs/reference/query-dsl/regexp-syntax.asciidoc
@@ -220,12 +220,20 @@ Complement::
 --
 
 The complement is probably the most useful option. The shortest pattern that
-follows a tilde `"~"` is negated.  For the string `"abcdef"`:
+follows a tilde `"~"` is negated.  For instance, `"ab~cd" means:
+
+* Starts with `a`
+* Followed by `b`
+* Followed by a string of any length that it anything but `c`
+* Ends with `d`
+
+For the string `"abcdef"`:
 
     ab~df     # match
-    ab~cf     # no match
-    a~(cd)f   # match
-    a~(bc)f   # no match
+    ab~cf     # match
+    ab~cdef   # no match
+    a~(cb)def # match
+    a~(bc)def # no match
 
 Enabled with the `COMPLEMENT` or `ALL` flags.
 
diff --git a/docs/reference/query-dsl/special-queries.asciidoc b/docs/reference/query-dsl/special-queries.asciidoc
index 1a2d63d2265..16d0020bf10 100644
--- a/docs/reference/query-dsl/special-queries.asciidoc
+++ b/docs/reference/query-dsl/special-queries.asciidoc
@@ -20,6 +20,10 @@ final query to execute.
 This query allows a script to act as a filter.  Also see the
 <<query-dsl-function-score-query,`function_score` query>>.
 
+<<query-dsl-percolator-query,`percolator` query>>::
+
+This query finds queries that are stored as documents that match with
+the specified document.
 
 include::mlt-query.asciidoc[]
 
@@ -27,3 +31,5 @@ include::template-query.asciidoc[]
 
 include::script-query.asciidoc[]
 
+include::percolator-query.asciidoc[]
+
diff --git a/docs/reference/query-dsl/template-query.asciidoc b/docs/reference/query-dsl/template-query.asciidoc
index 104adffe630..80155671459 100644
--- a/docs/reference/query-dsl/template-query.asciidoc
+++ b/docs/reference/query-dsl/template-query.asciidoc
@@ -106,7 +106,7 @@ GET /_search
     }
 }
 ------------------------------------------
-<1> Name of the query template in `config/scripts/`, i.e., `storedTemplate.mustache`.
+<1> Name of the query template in `config/scripts/`, i.e., `my_template.mustache`.
 
 
 There is also a dedicated `template` endpoint, allows you to template an entire search request.
diff --git a/docs/reference/query-dsl/term-query.asciidoc b/docs/reference/query-dsl/term-query.asciidoc
index 85608ca3aa5..801abf65af8 100644
--- a/docs/reference/query-dsl/term-query.asciidoc
+++ b/docs/reference/query-dsl/term-query.asciidoc
@@ -49,13 +49,13 @@ GET /_search
 .Why doesn't the `term` query match my document?
 **************************************************
 
-String fields can be `analyzed` (treated as full text, like the body of an
-email), or `not_analyzed` (treated as exact values, like an email address or a
-zip code).  Exact values (like numbers, dates, and `not_analyzed` strings) have
+String fields can be of type `text` (treated as full text, like the body of an
+email), or `keyword` (treated as exact values, like an email address or a
+zip code).  Exact values (like numbers, dates, and keywords) have
 the exact value specified in the field added to the inverted index in order
 to make them searchable.
 
-By default, however, `string` fields are `analyzed`. This means that their
+However, `text` fields are `analyzed`. This means that their
 values are first passed through an <<analysis,analyzer>> to produce a list of
 terms, which are then added to the inverted index.
 
@@ -70,7 +70,7 @@ within a big block of full text.
 
 The `term` query looks for the *exact* term in the field's inverted index --
 it doesn't know anything about the field's analyzer.  This makes it useful for
-looking up values in `not_analyzed` string fields, or in numeric or date
+looking up values in keyword fields, or in numeric or date
 fields.  When querying full text fields, use the
 <<query-dsl-match-query,`match` query>> instead, which understands how the field
 has been analyzed.
@@ -86,11 +86,10 @@ PUT my_index
     "my_type": {
       "properties": {
         "full_text": {
-          "type":  "string" <1>
+          "type":  "text" <1>
         },
         "exact_value": {
-          "type":  "string",
-          "index": "not_analyzed" <2>
+          "type":  "keyword" <2>
         }
       }
     }
@@ -105,8 +104,8 @@ PUT my_index/my_type/1
 --------------------------------------------------
 // AUTOSENSE
 
-<1> The `full_text` field is `analyzed` by default.
-<2> The `exact_value` field is set to be `not_analyzed`.
+<1> The `full_text` field is of type `text` and will be analyzed.
+<2> The `exact_value` field is of type `keyword` and will NOT be analyzed.
 <3> The `full_text` inverted index will contain the terms: [`quick`, `foxes`].
 <4> The `exact_value` inverted index will contain the exact term: [`Quick Foxes!`].
 
diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc
index 267525b1b3c..f9391cece06 100644
--- a/docs/reference/release-notes.asciidoc
+++ b/docs/reference/release-notes.asciidoc
@@ -3,5 +3,11 @@
 
 [partintro]
 --
-This section will summarize the changes in released versions.
+This section summarizes the changes in each release.
+
+* <<release-notes-5.0.0-alpha1>>
+* <<release-notes-5.0.0-alpha1-2x>>
+
 --
+include::release-notes/5.0.0-alpha1.asciidoc[]
+include::release-notes/5.0.0-alpha1-2x.asciidoc[]
diff --git a/docs/reference/release-notes/5.0.0-alpha1-2x.asciidoc b/docs/reference/release-notes/5.0.0-alpha1-2x.asciidoc
new file mode 100644
index 00000000000..061f3ae4839
--- /dev/null
+++ b/docs/reference/release-notes/5.0.0-alpha1-2x.asciidoc
@@ -0,0 +1,600 @@
+[[release-notes-5.0.0-alpha1-2x]]
+== 5.0.0-alpha1 Release Notes (Changes previously released in 2.x)
+
+The changes listed below were first released in the 2.x series. Changes
+released for the first time in Elasticsearch 5.0.0-alpha1 are listed in
+<<release-notes-5.0.0-alpha1>>.
+
+
+
+[[breaking-5.0.0-alpha1-2x]]
+[float]
+=== Breaking changes
+
+Allocation::
+* Speed up shard balancer by reusing shard model while moving shards that can no longer be allocated to a node {pull}16926[#16926]
+
+Index APIs::
+* Change Field stats API response to include both number and string based min and max values {pull}14674[#14674] (issue: {issue}14404[#14404])
+* Add Force Merge API, deprecate Optimize API {pull}13778[#13778]
+
+Internal::
+* Forbid changing thread pool types {pull}14367[#14367] (issues: {issue}14294[#14294], {issue}2509[#2509], {issue}2858[#2858], {issue}5152[#5152])
+
+Logging::
+* Log cluster health status changes {pull}14557[#14557] (issue: {issue}11657[#11657])
+
+Mapping::
+* Add per-index setting to limit number of nested fields {pull}15989[#15989] (issue: {issue}14983[#14983])
+
+Nested Docs::
+* If sorting by nested field then the `nested_path` should always be specified {pull}13429[#13429] (issue: {issue}13420[#13420])
+
+Scripting::
+* Filter classes loaded by scripts {pull}15262[#15262]
+* Lock down javascript and python script engines better {pull}13924[#13924]
+
+Search::
+* Limit the size of the result window to a dynamic property {pull}13188[#13188] (issue: {issue}9311[#9311])
+
+
+
+[[feature-5.0.0-alpha1-2x]]
+[float]
+=== New features
+
+Aggregations::
+* Adds geo_centroid metric aggregator {pull}13846[#13846] (issue: {issue}13621[#13621])
+* Add `percentiles_bucket` pipeline aggregation {pull}13186[#13186]
+* Add `stats_bucket` / `extended_stats_bucket` pipeline aggs {pull}13128[#13128]
+
+Geo::
+* Add CONTAINS relation to geo_shape query {pull}14810[#14810] (issue: {issue}14713[#14713])
+* Add support for Lucene 5.4 GeoPoint queries {pull}14537[#14537]
+* Add GeoPointV2 Field Mapping {pull}14536[#14536]
+
+Network::
+* Allow binding to multiple addresses. {pull}13954[#13954] (issue: {issue}13592[#13592])
+
+Plugin Analysis Phonetic::
+* Add support for `daitch_mokotoff` {pull}14834[#14834]
+
+Plugin Repository S3::
+* Add support for S3 storage class {pull}13656[#13656] (issue: {issue}13655[#13655])
+
+Plugins::
+* Decentralize plugin security {pull}14108[#14108]
+
+Search::
+* Add query profiler {pull}14889[#14889] (issues: {issue}12974[#12974], {issue}6699[#6699])
+
+
+
+[[enhancement-5.0.0-alpha1-2x]]
+[float]
+=== Enhancements
+
+Aliases::
+* Add support to _aliases endpoint to specify multiple indices and aliases in one action {pull}15305[#15305] (issue: {issue}15186[#15186])
+
+Allocation::
+* Skip capturing least/most FS info for an FS with no total {pull}16001[#16001] (issue: {issue}15919[#15919])
+* Speed improvements for BalancedShardsAllocator {pull}15678[#15678] (issue: {issue}6372[#6372])
+* Simplify delayed shard allocation {pull}14808[#14808]
+* Add cluster-wide setting for total shard limit {pull}14563[#14563] (issue: {issue}14456[#14456])
+* Early terminate high disk watermark checks on single data node cluster {pull}13882[#13882] (issue: {issue}9391[#9391])
+* Also use PriorityComparator in shard balancer {pull}13256[#13256] (issue: {issue}13249[#13249])
+* Add support for filtering by publish IP address {pull}8801[#8801]
+
+Analysis::
+* Add detail response support for _analyze API {pull}11660[#11660] (issue: {issue}11076[#11076])
+
+CAT API::
+* Add sync_id to cat shards API {pull}14712[#14712] (issue: {issue}14705[#14705])
+* Add duration field to /_cat/snapshots {pull}14385[#14385]
+* Add cat API for repositories and snapshots {pull}14247[#14247] (issue: {issue}13919[#13919])
+* Adds disk used by indices to _cat/allocation {pull}13783[#13783] (issue: {issue}13529[#13529])
+
+CRUD::
+
+Cluster::
+* Shard state action request logging {pull}16396[#16396]
+* Safe cluster state task notifications {pull}15777[#15777]
+* Reroute once per batch of shard failures {pull}15510[#15510]
+* Add callback for publication of new cluster state {pull}15494[#15494] (issue: {issue}15482[#15482])
+* Use general cluster state batching mechanism for shard started {pull}15023[#15023] (issues: {issue}14725[#14725], {issue}14899[#14899])
+* Use general cluster state batching mechanism for shard failures {pull}15016[#15016] (issues: {issue}14725[#14725], {issue}14899[#14899])
+* Set an newly created IndexShard's ShardRouting before exposing it to operations {pull}14918[#14918] (issue: {issue}10708[#10708])
+* Uniform exceptions for TransportMasterNodeAction {pull}14737[#14737]
+
+Core::
+* Remove log4j exception hiding {pull}16834[#16834]
+* Avoid cloning MessageDigest instances {pull}16479[#16479]
+* Add a hard check to ensure we are running with the expected lucene version {pull}16305[#16305] (issue: {issue}16301[#16301])
+* If we can't get a MAC address for the node, use a dummy one {pull}15266[#15266] (issue: {issue}10099[#10099])
+* Simplify shard inactive logging {pull}15259[#15259] (issue: {issue}15252[#15252])
+* Simplify IndexingMemoryController#checkIdle {pull}15252[#15252] (issue: {issue}15251[#15251])
+* IndexingMemoryController should not track shard index states {pull}15251[#15251] (issues: {issue}13918[#13918], {issue}15225[#15225])
+* Verify Checksum once it has been fully written to fail as soon as possible {pull}13896[#13896]
+
+Discovery::
+* Don't allow nodes with missing custom meta data to join cluster {pull}15401[#15401] (issue: {issue}13445[#13445])
+
+Exceptions::
+* Added file name to exceptions when failing to read index state {pull}16850[#16850] (issue: {issue}16713[#16713])
+* Add Exception class name to message in `NotSerializableExceptionWrapper` {pull}16325[#16325]
+* Deduplicate cause if already contained in shard failures {pull}14432[#14432]
+* Give a better exception when running from freebsd jail without enforce_statfs=1 {pull}14135[#14135] (issue: {issue}12018[#12018])
+* Make root_cause of field conflicts more obvious {pull}13976[#13976] (issue: {issue}12839[#12839])
+* Use a dedicated id to serialize EsExceptions instead of it's class name. {pull}13629[#13629]
+
+Fielddata::
+* Update GeoPoint FieldData for GeoPointV2 {pull}14345[#14345]
+
+Geo::
+* Upgrade GeoPointField to use Lucene 5.5 PrefixEncoding {pull}16482[#16482]
+* Geo: Fix toString() in GeoDistanceRangeQuery and GeoPolygonQuery {pull}15026[#15026]
+* Enable GeoPointV2 with backward compatibility testing {pull}14667[#14667] (issues: {issue}10761[#10761], {issue}11159[#11159], {issue}9859[#9859])
+* Refactor Geo utilities to Lucene 5.4 {pull}14339[#14339]
+
+Index APIs::
+* Add option to disable closing indices {pull}14169[#14169] (issue: {issue}14168[#14168])
+
+Index Templates::
+* Disallow index template pattern to be the same as an alias name {pull}15184[#15184] (issue: {issue}14842[#14842])
+
+Internal::
+* Cleanup search sub transport actions and collapse o.e.action.search.type package into o.e.action.search {pull}16758[#16758] (issue: {issue}11710[#11710])
+* Simplify the Text API. {pull}15511[#15511]
+* Simpler using compressed oops flag representation {pull}15509[#15509] (issue: {issue}15489[#15489])
+* Info on compressed ordinary object pointers {pull}15489[#15489] (issues: {issue}13187[#13187], {issue}455[#455])
+* Explicitly log cluster state update failures {pull}15428[#15428] (issues: {issue}14899[#14899], {issue}15016[#15016], {issue}15023[#15023])
+* Use transport service to handle RetryOnReplicaException to execute replica action on the current node {pull}15363[#15363]
+* Make IndexShard operation be more explicit about whether they are expected to run on a primary or replica {pull}15282[#15282]
+* Avoid trace logging allocations in TransportBroadcastByNodeAction {pull}15221[#15221]
+* Only trace log shard not available exceptions {pull}14950[#14950] (issue: {issue}14927[#14927])
+* Transport options should be immutable {pull}14760[#14760]
+* Fix dangling comma in ClusterBlock#toString {pull}14483[#14483]
+* Improve some logging around master election and cluster state {pull}14481[#14481]
+* Add System#exit(), Runtime#exit() and Runtime#halt() to forbidden APIs {pull}14473[#14473] (issue: {issue}12596[#12596])
+* Simplify XContent detection. {pull}14472[#14472]
+* Add threadgroup isolation. {pull}14353[#14353]
+* Cleanup plugin security {pull}14311[#14311]
+* Add workaround for JDK-8014008 {pull}14274[#14274]
+* Refactor retry logic for TransportMasterNodeAction {pull}14222[#14222]
+* Remove MetaDataSerivce and it's semaphores {pull}14159[#14159] (issue: {issue}1296[#1296])
+* Cleanup IndexMetaData {pull}14119[#14119]
+* Add SpecialPermission to guard exceptions to security policy. {pull}13854[#13854]
+* Clean up scripting permissions. {pull}13844[#13844]
+* Factor groovy out of core into lang-groovy {pull}13834[#13834] (issue: {issue}13725[#13725])
+* More helpful error message on parameter order {pull}13737[#13737]
+* Factor expressions scripts out to lang-expression plugin {pull}13726[#13726] (issue: {issue}13725[#13725])
+* Cleanup InternalClusterInfoService {pull}13543[#13543]
+* Remove and forbid use of com.google.common.base.Throwables {pull}13409[#13409] (issue: {issue}13224[#13224])
+* Remove cyclic dependencies between IndexService and FieldData / BitSet caches {pull}13381[#13381]
+* Remove and forbid use of com.google.common.base.Objects {pull}13355[#13355] (issue: {issue}13224[#13224])
+* Enable indy (invokedynamic) compile flag for Groovy scripts by default {pull}8201[#8201] (issue: {issue}8184[#8184])
+
+Java API::
+* Align handling of interrupts in BulkProcessor {pull}15527[#15527] (issue: {issue}14833[#14833])
+* BulkProcessor backs off exponentially by default {pull}15513[#15513] (issue: {issue}14829[#14829])
+* Reject refresh usage in bulk items when using and fix NPE when no source {pull}15082[#15082] (issue: {issue}7361[#7361])
+* BulkProcessor retries after request handling has been rejected due to a full thread pool {pull}14829[#14829] (issue: {issue}14620[#14620])
+
+Logging::
+* Log suppressed stack traces under DEBUG {pull}16627[#16627] (issues: {issue}12991[#12991], {issue}15329[#15329], {issue}16622[#16622])
+* Add circuit breaker name to logging package {pull}14661[#14661]
+* Move logging for the amount of free disk to TRACE {pull}14403[#14403] (issue: {issue}12843[#12843])
+* Map log-level 'trace' to JDK-Level 'FINEST' {pull}14234[#14234]
+
+Mapping::
+* Expose the reason why a mapping merge is issued. {pull}16059[#16059] (issue: {issue}15989[#15989])
+* Add sub-fields support to `bool` fields. {pull}15636[#15636] (issue: {issue}6587[#6587])
+* Improve cross-type dynamic mapping updates. {pull}15633[#15633] (issue: {issue}15568[#15568])
+* Make mapping updates more robust. {pull}15539[#15539]
+* Make mapping serialization more robust. {pull}15480[#15480]
+* Make mappings immutable. {pull}15313[#15313] (issue: {issue}9365[#9365])
+* Make MappedFieldType.checkTypeName part of MappedFieldType.checkCompatibility. {pull}15245[#15245]
+* Register field mappers at the node level. {pull}14896[#14896] (issue: {issue}14828[#14828])
+
+Network::
+* Provide better error message when an incompatible node connects to a node {pull}17182[#17182] (issue: {issue}17090[#17090])
+* Add additional fallback to http.publish_port and restrict fallback to transport.publish_port {pull}16626[#16626] (issue: {issue}14535[#14535])
+* only allow code to bind to the user's configured port numbers/ranges {pull}14549[#14549]
+* Port of publishAddress should match port of corresponding boundAddress {pull}14535[#14535] (issues: {issue}14503[#14503], {issue}14513[#14513], {issue}14514[#14514])
+
+Packaging::
+* Windows service: Use JAVA_HOME environment variable in registry {pull}16552[#16552] (issue: {issue}13521[#13521])
+* Default standard output to the journal in systemd {pull}16159[#16159] (issues: {issue}15315[#15315], {issue}16134[#16134])
+* Use egrep instead of grep -E for Solaris {pull}15755[#15755] (issue: {issue}15628[#15628])
+* punch thru symlinks when loading plugins/modules {pull}15311[#15311]
+* set ActiveProcessLimit=1 on windows {pull}15055[#15055]
+* set RLIMIT_NPROC = 0 on bsd/os X systems. {pull}15039[#15039]
+* Drop ability to execute on Solaris {pull}14200[#14200]
+* Nuke ES_CLASSPATH appending, JarHell fail on empty classpath elements {pull}13880[#13880] (issues: {issue}13812[#13812], {issue}13864[#13864])
+* improve seccomp syscall filtering {pull}13829[#13829]
+* Block process execution with seccomp on linux/amd64 {pull}13753[#13753]
+* Get lang-javascript, lang-python, securemock ready for script refactoring {pull}13695[#13695]
+* Remove some bogus permissions only needed for tests. {pull}13620[#13620]
+* Remove java.lang.reflect.ReflectPermission "suppressAccessChecks" {pull}13603[#13603]
+* Remove JAVA_HOME detection from the debian init script {pull}13514[#13514] (issues: {issue}13403[#13403], {issue}9774[#9774])
+
+Plugin Cloud GCE::
+* cloud-gce plugin should check `discovery.type` {pull}13809[#13809] (issue: {issue}13614[#13614])
+* Adding backoff from retries on GCE errors {pull}13671[#13671] (issue: {issue}13460[#13460])
+
+Plugin Discovery EC2::
+* Add ap-northeast-2 (seoul) endpoints for EC2 discovery and S3 snapshots {pull}16167[#16167] (issue: {issue}16166[#16166])
+* Adding US-Gov-West {pull}14358[#14358]
+* Improved building of disco nodes {pull}14155[#14155]
+
+Plugin Ingest Attachment::
+* Fix attachments plugins with docx {pull}17059[#17059] (issue: {issue}16864[#16864])
+
+Plugin Repository Azure::
+* Add support for secondary azure storage account {pull}13779[#13779] (issue: {issue}13228[#13228])
+
+Plugin Repository S3::
+* Add aws canned acl {pull}14297[#14297] (issue: {issue}14103[#14103])
+* Enable S3SignerType {pull}13360[#13360] (issue: {issue}13332[#13332])
+
+Plugins::
+* Expose http.type setting, and collapse al(most all) modules relating to transport/http {pull}15434[#15434] (issue: {issue}14148[#14148])
+* Ban RuntimePermission("getClassLoader") {pull}15253[#15253]
+* Add nicer error message when a plugin descriptor is missing {pull}15200[#15200] (issue: {issue}15197[#15197])
+* Don't be lenient in PluginService#processModule(Module) {pull}14306[#14306]
+* Adds a validation for plugins script to check if java is set {pull}13633[#13633] (issue: {issue}13613[#13613])
+* Output plugin info only in verbose mode {pull}12908[#12908] (issue: {issue}12907[#12907])
+
+Query DSL::
+* Allow CIDR notation in query string query {pull}14773[#14773] (issue: {issue}7464[#7464])
+* Internal: simplify filtered query conversion to lucene query {pull}13312[#13312] (issue: {issue}13272[#13272])
+
+REST::
+* Make XContentGenerator.writeRaw* safer. {pull}15358[#15358]
+* Filter path refactoring {pull}14390[#14390] (issues: {issue}10980[#10980], {issue}11560[#11560], {issue}13344[#13344])
+
+Recovery::
+* Handle cancel exceptions on recovery target if the cancel comes from the source {pull}15309[#15309]
+* Decouple routing and primary operation logic in TransportReplicationAction {pull}14852[#14852]
+
+Reindex API::
+* Implement helpful interfaces in reindex requests {pull}17032[#17032]
+* Reindex should timeout if sub-requests timeout {pull}16962[#16962]
+* Teach reindex to retry on rejection {pull}16556[#16556] (issue: {issue}16093[#16093])
+
+Scripting::
+* Remove suppressAccessChecks permission for Groovy script plugin {pull}16839[#16839] (issue: {issue}16527[#16527])
+* Class permission for Groovy references {pull}16660[#16660] (issue: {issue}16657[#16657])
+* Scripting: Allow to get size of array in mustache {pull}16193[#16193]
+* Enhancements to the mustache script engine {pull}15661[#15661]
+* Add property permissions so groovy scripts can serialize json {pull}14500[#14500] (issue: {issue}14488[#14488])
+* Remove ScriptEngineService.unwrap. {pull}13958[#13958]
+* Remove ScriptEngineService.execute. {pull}13956[#13956]
+
+Search::
+* Caching Weight wrappers should propagate the BulkScorer. {pull}14317[#14317]
+* fix numerical issue in function score query {pull}14085[#14085]
+* Optimize scrolls for constant-score queries. {pull}13311[#13311]
+
+Settings::
+* Log warning if max file descriptors too low {pull}16506[#16506]
+
+Snapshot/Restore::
+* Support wildcards for getting repositories and snapshots {pull}15151[#15151] (issue: {issue}4758[#4758])
+* Add ignore_unavailable parameter to skip unavailable snapshot {pull}14471[#14471] (issue: {issue}13887[#13887])
+* Simplify the BlobContainer blob writing interface {pull}13434[#13434]
+
+Stats::
+* Pull Fields instance once from LeafReader in completion stats {pull}15090[#15090] (issue: {issue}6593[#6593])
+* Add os.allocated_processors stats {pull}14409[#14409] (issue: {issue}13917[#13917])
+* Adds stats counter for failed indexing requests {pull}13130[#13130] (issue: {issue}8938[#8938])
+
+Top Hits::
+* Put method addField on TopHitsBuilder {pull}14597[#14597] (issue: {issue}12962[#12962])
+
+Translog::
+* Check for tragic event on all kinds of exceptions not only ACE and IOException {pull}15535[#15535]
+
+Tribe Node::
+* Tribe nodes should apply cluster state updates in batches {pull}14993[#14993] (issues: {issue}14725[#14725], {issue}14899[#14899])
+
+
+
+[[bug-5.0.0-alpha1-2x]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Build empty extended stats aggregation if no docs collected for bucket {pull}16972[#16972] (issues: {issue}16812[#16812], {issue}9544[#9544])
+* Set meta data for pipeline aggregations {pull}16516[#16516] (issue: {issue}16484[#16484])
+* Filter(s) aggregation should create weights only once. {pull}15998[#15998]
+* Make `missing` on terms aggs work with all execution modes. {pull}15746[#15746] (issue: {issue}14882[#14882])
+* Run pipeline aggregations for empty buckets added in the Range Aggregation {pull}15519[#15519] (issue: {issue}15471[#15471])
+* [Children agg] fix bug that prevented all child docs from being evaluated {pull}15457[#15457]
+* Correct typo in class name of StatsAggregator {pull}15321[#15321] (issue: {issue}14730[#14730])
+* Fix significant terms reduce for long terms {pull}14948[#14948] (issue: {issue}13522[#13522])
+* Fix NPE in Derivative Pipeline when current bucket value is null {pull}14745[#14745]
+* Pass extended bounds into HistogramAggregator when creating an unmapped aggregator {pull}14742[#14742] (issue: {issue}14735[#14735])
+* Added correct generic type parameter on ScriptedMetricBuilder {pull}14018[#14018] (issue: {issue}13986[#13986])
+* Pipeline Aggregations at the root of the agg tree are now validated {pull}13475[#13475] (issue: {issue}13179[#13179])
+
+Aliases::
+* Fix _aliases filter and null parameters {pull}16553[#16553] (issues: {issue}16547[#16547], {issue}16549[#16549])
+
+Allocation::
+* IndicesStore checks for `allocated elsewhere` for every shard not alocated on the local node {pull}17106[#17106]
+* Prevent peer recovery from node with older version {pull}15775[#15775]
+* Fix calculation of next delay for delayed shard allocation {pull}14765[#14765]
+* Take ignored unallocated shards into account when making allocation decision {pull}14678[#14678] (issue: {issue}14670[#14670])
+* Only allow rebalance operations to run if all shard store data is available {pull}14591[#14591] (issue: {issue}14387[#14387])
+* Delayed allocation can miss a reroute {pull}14494[#14494] (issues: {issue}14010[#14010], {issue}14011[#14011], {issue}14445[#14445])
+* Check rebalancing constraints when shards are moved from a node they can no longer remain on {pull}14259[#14259] (issue: {issue}14057[#14057])
+
+Analysis::
+* Analysis : Allow string explain param in JSON {pull}16977[#16977] (issue: {issue}16925[#16925])
+* Analysis : Fix no response from Analyze API without specified index {pull}15447[#15447] (issue: {issue}15148[#15148])
+
+Bulk::
+* Bulk api: fail deletes when routing is required but not specified {pull}16675[#16675] (issues: {issue}10136[#10136], {issue}16645[#16645])
+* Do not release unacquired semaphore {pull}14909[#14909] (issue: {issue}14908[#14908])
+
+CAT API::
+* Properly set indices and indicesOptions on subrequest made by /_cat/indices {pull}14360[#14360]
+
+CRUD::
+* Throw exception if content type could not be determined in Update API {pull}15904[#15904] (issue: {issue}15822[#15822])
+* Index name expressions should not be broken up {pull}13691[#13691] (issue: {issue}13665[#13665])
+
+Cache::
+* Handle closed readers in ShardCoreKeyMap {pull}16027[#16027]
+
+Cluster::
+* Index deletes not applied when cluster UUID has changed {pull}16825[#16825] (issue: {issue}11665[#11665])
+* Only fail the relocation target when a replication request on it fails {pull}15791[#15791] (issue: {issue}15790[#15790])
+* Handle shards assigned to nodes that are not in the cluster state {pull}14586[#14586] (issue: {issue}14584[#14584])
+* Bulk cluster state updates on index deletion {pull}11258[#11258] (issue: {issue}7295[#7295])
+
+Core::
+* BitSetFilterCache duplicates its content. {pull}15836[#15836] (issue: {issue}15820[#15820])
+* Limit the max size of bulk and index thread pools to bounded number of processors {pull}15585[#15585] (issue: {issue}15582[#15582])
+* AllTermQuery's scorer should skip segments that never saw the requested term {pull}15506[#15506]
+* Include root-cause exception when we fail to change shard's index buffer {pull}14867[#14867]
+* Restore thread interrupt flag after an InterruptedException {pull}14799[#14799] (issue: {issue}14798[#14798])
+* Record all bytes of the checksum in VerifyingIndexOutput {pull}13923[#13923] (issues: {issue}13848[#13848], {issue}13896[#13896])
+* When shard becomes active again, immediately increase its indexing buffer {pull}13918[#13918] (issue: {issue}13802[#13802])
+* Close TokenStream in finally clause {pull}13870[#13870] (issue: {issue}11947[#11947])
+* LoggingRunnable.run should catch and log all errors, not just Exception? {pull}13718[#13718] (issue: {issue}13487[#13487])
+
+Exceptions::
+* Fix ensureNodesAreAvailable's error message {pull}14007[#14007] (issue: {issue}13957[#13957])
+
+Expressions::
+* Check that _value is used in aggregations script before setting value to specialValue {pull}17091[#17091] (issue: {issue}14262[#14262])
+
+Fielddata::
+* Don't cache top level field data for fields that don't exist {pull}14693[#14693]
+
+Geo::
+* Remove .geohash suffix from GeoDistanceQuery and GeoDistanceRangeQuery {pull}15871[#15871] (issue: {issue}15179[#15179])
+* Geo: Allow numeric parameters enclosed in quotes for 'geohash_grid' aggregation {pull}14440[#14440] (issue: {issue}13132[#13132])
+* Resync Geopoint hashCode/equals method {pull}14124[#14124] (issue: {issue}14083[#14083])
+* Fix GeoPointFieldMapper to index geohash at correct precision. {pull}13649[#13649] (issue: {issue}12467[#12467])
+
+Highlighting::
+* Don't override originalQuery with request filters {pull}15793[#15793] (issue: {issue}15689[#15689])
+* Fix spans extraction to not also include individual terms. {pull}15516[#15516] (issues: {issue}13239[#13239], {issue}15291[#15291])
+
+Index APIs::
+* Field stats: Index constraints should remove indices in the response if the field to evaluate is empty {pull}14868[#14868]
+* Field stats: Fix NPE for index constraint on empty index {pull}14841[#14841]
+* Field stats: Added `format` option for index constraints {pull}14823[#14823] (issue: {issue}14804[#14804])
+* Forbid index name `.` and `..` {pull}13862[#13862] (issue: {issue}13858[#13858])
+
+Inner Hits::
+* Query and top level inner hit definitions shouldn't overwrite each other {pull}16222[#16222] (issue: {issue}16218[#16218])
+
+Internal::
+* Log uncaught exceptions from scheduled once tasks {pull}15824[#15824] (issue: {issue}15814[#15814])
+* FunctionScoreQuery should implement two-phase iteration. {pull}15602[#15602]
+* Make sure the remaining delay of unassigned shard is updated with every reroute {pull}14890[#14890] (issue: {issue}14808[#14808])
+* Throw a meaningful error when loading metadata and an alias and index have the same name {pull}14842[#14842] (issue: {issue}14706[#14706])
+* fixup issues with 32-bit jvm {pull}14609[#14609]
+* Failure to update the cluster state with the recovered state should make sure it will be recovered later {pull}14485[#14485]
+* Gateway: a race condition can prevent the initial cluster state from being recovered {pull}13997[#13997]
+* Verify actually written checksum in VerifyingIndexOutput {pull}13848[#13848]
+* An inactive shard is activated by triggered synced flush {pull}13802[#13802]
+* Remove all setAccessible in tests and forbid {pull}13539[#13539]
+* Remove easy uses of setAccessible in tests. {pull}13537[#13537]
+* Ban setAccessible from core code, restore monitoring stats under java 9 {pull}13531[#13531] (issue: {issue}13527[#13527])
+
+Logging::
+* Add missing index name to indexing slow log {pull}17026[#17026] (issue: {issue}17025[#17025])
+* ParseFieldMatcher should log when using deprecated settings. {pull}16988[#16988]
+* Don't log multi-megabyte guice exceptions. {pull}13782[#13782]
+* Moving system property setting to before it can be used {pull}13660[#13660] (issue: {issue}13658[#13658])
+
+Mapping::
+* Put mapping operations must update metadata of all types. {pull}16264[#16264] (issue: {issue}16239[#16239])
+* Fix serialization of `search_analyzer`. {pull}16255[#16255]
+* Reuse metadata mappers for dynamic updates. {pull}16023[#16023] (issue: {issue}15997[#15997])
+* Fix MapperService#searchFilter(...) {pull}15923[#15923] (issue: {issue}15757[#15757])
+* Fix initial sizing of BytesStreamOutput. {pull}15864[#15864] (issue: {issue}15789[#15789])
+* MetaDataMappingService should call MapperService.merge with the original mapping update. {pull}15508[#15508]
+* MapperService: check index.mapper.dynamic during index creation {pull}15424[#15424] (issue: {issue}15381[#15381])
+* Only text fields should accept analyzer and term vector settings. {pull}15308[#15308]
+* Mapper parsers should not check for a `tokenized` property. {pull}15289[#15289]
+* Validate that fields are defined only once. {pull}15243[#15243] (issue: {issue}15057[#15057])
+* Check mapping compatibility up-front. {pull}15175[#15175] (issue: {issue}15049[#15049])
+* Don't treat _default_ as a regular type. {pull}15156[#15156] (issue: {issue}15049[#15049])
+* Don't ignore mapping merge failures. {pull}15144[#15144] (issue: {issue}15049[#15049])
+* Treat mappings at an index-level feature. {pull}15142[#15142]
+* Make _type use doc values {pull}14783[#14783] (issue: {issue}14781[#14781])
+
+Network::
+* Only accept transport requests after node is fully initialized {pull}16746[#16746] (issue: {issue}16723[#16723])
+
+Packaging::
+* Fix waiting for pidfile {pull}16718[#16718] (issue: {issue}16717[#16717])
+* Fix Windows service installation failure {pull}15549[#15549] (issue: {issue}15349[#15349])
+* Enable es_include at init {pull}15173[#15173]
+* Handle system policy correctly {pull}14704[#14704] (issue: {issue}14690[#14690])
+* Startup script exit status should catch daemonized startup failures {pull}14170[#14170] (issue: {issue}14163[#14163])
+* Don't let ubuntu try to install its crazy jayatana agent. {pull}13813[#13813] (issue: {issue}13785[#13785])
+
+Parent/Child::
+* Check that parent_type in Has Parent Query has child types {pull}16923[#16923] (issue: {issue}16692[#16692])
+* Has child query forces default similarity {pull}16611[#16611] (issues: {issue}16550[#16550], {issue}4977[#4977])
+
+Percolator::
+* Don't replace found fields if map unmapped fields as string is enabled {pull}16043[#16043] (issue: {issue}10500[#10500])
+* mpercolate api should serialise start time  {pull}15938[#15938] (issue: {issue}15908[#15908])
+
+Plugin Delete By Query::
+* Fix Delete-by-Query with Shield {pull}14658[#14658] (issue: {issue}14527[#14527])
+
+Plugin Discovery GCE::
+* Add setFactory permission to GceDiscoveryPlugin {pull}16860[#16860] (issue: {issue}16485[#16485])
+
+Plugin Mapper Attachment::
+* Fix toXContent() for mapper attachments field {pull}15110[#15110]
+
+Plugin Repository Azure::
+
+Plugin Repository S3::
+* Hack around aws security hole of accessing sun.security.ssl, s3 repository works on java 9 again {pull}13538[#13538] (issue: {issue}432[#432])
+
+Plugins::
+* Fix plugin list command error message {pull}14288[#14288] (issue: {issue}14287[#14287])
+* Fix HTML response during redirection {pull}11374[#11374] (issue: {issue}11370[#11370])
+
+Query DSL::
+* Fix FunctionScore equals/hashCode to include minScore and friends {pull}15676[#15676]
+* Min should match greater than the number of optional clauses should return no result {pull}15571[#15571] (issue: {issue}15521[#15521])
+* Return a better exception message when `regexp` query is used on a numeric field {pull}14910[#14910] (issue: {issue}14782[#14782])
+
+REST::
+* Remove detect_noop from REST spec {pull}16386[#16386]
+* Make text parsing less lenient. {pull}15679[#15679]
+* Throw exception when trying to write map with null keys {pull}15479[#15479] (issue: {issue}14346[#14346])
+* Fix OOM in AbstractXContentParser {pull}15350[#15350] (issue: {issue}15338[#15338])
+* XContentFactory.xContentType: allow for possible UTF-8 BOM for JSON XContentType {pull}14611[#14611] (issue: {issue}14442[#14442])
+* RestUtils.decodeQueryString ignores the URI fragment when parsing a query string {pull}13365[#13365] (issue: {issue}13320[#13320])
+
+Recovery::
+* Try to renew sync ID if `flush=true` on forceMerge {pull}17108[#17108] (issue: {issue}17019[#17019])
+* CancellableThreads should also treat ThreadInterruptedException as InterruptedException {pull}15318[#15318]
+
+Reindex API::
+* Properly register reindex status {pull}17125[#17125]
+* Make search failure cause rest failure {pull}16889[#16889] (issue: {issue}16037[#16037])
+
+Scripting::
+* Add permission to access sun.reflect.MethodAccessorImpl from Groovy scripts {pull}16540[#16540] (issue: {issue}16536[#16536])
+* Security permissions for Groovy closures {pull}16196[#16196] (issues: {issue}16194[#16194], {issue}248[#248])
+
+Search::
+* Do not apply minimum_should_match on auto generated boolean query if the coordination factor is disabled. {pull}16155[#16155]
+* Do not apply minimum-should-match on a boolean query if the coords are disabled {pull}16078[#16078] (issue: {issue}15858[#15858])
+* Fix blended terms take 2 {pull}15894[#15894] (issue: {issue}15860[#15860])
+* Fix NPE when a segment with an empty cache gets closed. {pull}15202[#15202] (issue: {issue}15043[#15043])
+* Fix the quotes in the explain message for a script score function without parameters {pull}11398[#11398]
+
+Settings::
+* TransportClient should use updated setting for initialization of modules and service {pull}16095[#16095]
+* ByteSizeValue.equals should normalize units {pull}13784[#13784]
+
+Snapshot/Restore::
+* Prevent closing index during snapshot restore {pull}16933[#16933] (issue: {issue}16321[#16321])
+* Add node version check to shard allocation during restore {pull}16520[#16520] (issue: {issue}16519[#16519])
+* Snapshot restore and index creates should keep index settings and cluster blocks in sync {pull}13931[#13931] (issue: {issue}13213[#13213])
+* Fix blob size in writeBlob() method {pull}13574[#13574] (issue: {issue}13434[#13434])
+
+Stats::
+* Fix recovery translog stats totals when recovering from store {pull}16493[#16493] (issue: {issue}15974[#15974])
+* Fix calculation of age of pending tasks {pull}15995[#15995] (issue: {issue}15988[#15988])
+* Add extra validation into `cluster/stats` {pull}14699[#14699] (issue: {issue}7390[#7390])
+* Omit current* stats for OldShardStats {pull}13801[#13801] (issue: {issue}13386[#13386])
+
+Translog::
+* Never delete translog-N.tlog file when creation fails {pull}15788[#15788]
+* Close recovered translog readers if createWriter fails {pull}15762[#15762] (issue: {issue}15754[#15754])
+* Fail and close translog hard if writing to disk fails {pull}15420[#15420] (issue: {issue}15333[#15333])
+* Prevent writing to closed channel if translog is already closed {pull}15012[#15012] (issue: {issue}14866[#14866])
+* Don't delete temp recovered checkpoint file if it was renamed {pull}14872[#14872] (issue: {issue}14695[#14695])
+* Translog recovery can repeatedly fail if we run out of disk {pull}14695[#14695]
+* Pending operations in the translog prevent shard from being marked as inactive {pull}13759[#13759] (issue: {issue}13707[#13707])
+
+Tribe Node::
+* Passthrough environment and network settings to tribe client nodes {pull}16893[#16893]
+* Tribe node: pass path.conf to inner tribe clients {pull}16258[#16258] (issue: {issue}16253[#16253])
+* Fix tribe node to load config file for internal client nodes {pull}15300[#15300] (issues: {issue}13383[#13383], {issue}14573[#14573])
+
+
+
+[[regression-5.0.0-alpha1-2x]]
+[float]
+=== Regressions
+
+Analysis::
+* Add PathHierarchy type back to path_hierarchy tokenizer for backward compatibility with 1.x {pull}15785[#15785] (issue: {issue}15756[#15756])
+
+Internal::
+* Deduplicate concrete indices after indices resolution {pull}14316[#14316] (issues: {issue}11258[#11258], {issue}12058[#12058])
+
+Plugin Cloud Azure::
+* Filter cloud azure credentials {pull}14863[#14863] (issues: {issue}13779[#13779], {issue}14843[#14843])
+
+REST::
+* Don't return all indices immediately if count of expressions >1 and first expression is * {pull}17033[#17033] (issue: {issue}17027[#17027])
+
+
+
+[[upgrade-5.0.0-alpha1-2x]]
+[float]
+=== Upgrades
+
+Core::
+* Upgrade to Lucene 5.5.0 official release {pull}16742[#16742]
+* Upgrade to lucene 5.5.0-snapshot-850c6c2 {pull}16615[#16615]
+* Upgrade to lucene 5.5.0-snapshot-4de5f1d {pull}16400[#16400] (issues: {issue}16373[#16373], {issue}16399[#16399])
+* Update lucene to r1725675 {pull}16114[#16114]
+* Upgrade to lucene-5.5.0-snapshot-1721183. {pull}15575[#15575]
+* Upgrade Lucene to 5.4.0-snapshot-1715952 {pull}14951[#14951]
+* Upgrade Lucene to 5.4.0-snapshot-1714615 {pull}14784[#14784]
+* Upgrade to lucene-5.4.0-snapshot-1712973. {pull}14619[#14619]
+* update to lucene-5.4.x-snapshot-1711508 {pull}14398[#14398]
+* Upgrade to lucene-5.4-snapshot-1710880. {pull}14320[#14320]
+* Upgrade to lucene-5.4-snapshot-1708254. {pull}14074[#14074]
+* upgrade lucene to r1702265 {pull}13439[#13439]
+* Upgrade master to lucene 5.4-snapshot r1701068 {pull}13324[#13324]
+
+Geo::
+* Update to spatial4j 0.5 for correct Multi-Geometry {pull}14269[#14269] (issue: {issue}9904[#9904])
+
+Internal::
+* Update to Jackson 2.6.2 {pull}13344[#13344] (issues: {issue}10980[#10980], {issue}207[#207], {issue}213[#213])
+
+Plugin Cloud AWS::
+* Update AWS SDK version to 1.10.19 {pull}13655[#13655] (issue: {issue}13656[#13656])
+
+Plugin Cloud Azure::
+* Update Azure Service Management API to 0.9.0 {pull}15232[#15232] (issue: {issue}15209[#15209])
+
+Plugin Discovery Azure::
+* Upgrade azure SDK to 0.9.3 {pull}17102[#17102] (issues: {issue}17042[#17042], {issue}557[#557])
+
+Plugin Lang JS::
+* upgrade rhino for plugins/lang-javascript {pull}14466[#14466]
+
+Plugin Repository Azure::
+* Upgrade Azure Storage client to 4.0.0 {pull}16084[#16084] (issues: {issue}12567[#12567], {issue}15080[#15080], {issue}15976[#15976])
+
+Plugin Repository S3::
+* Upgrade to aws 1.10.33 {pull}14672[#14672]
+
+Scripting::
+* Upgrade groovy dependency in lang-groovy module to version 2.4.6 {pull}16830[#16830] (issue: {issue}16527[#16527])
+
+
+
diff --git a/docs/reference/release-notes/5.0.0-alpha1.asciidoc b/docs/reference/release-notes/5.0.0-alpha1.asciidoc
new file mode 100644
index 00000000000..eac01a915e5
--- /dev/null
+++ b/docs/reference/release-notes/5.0.0-alpha1.asciidoc
@@ -0,0 +1,688 @@
+[[release-notes-5.0.0-alpha1]]
+== 5.0.0-alpha1 Release Notes
+
+The changes listed below have been released for the first time in
+Elasticsearch 5.0.0-alpha1.  Changes in this release which were first released
+in the 2.x series are listed in <<release-notes-5.0.0-alpha1-2x>>.
+
+[[breaking-5.0.0-alpha1]]
+[float]
+=== Breaking changes
+
+Aggregations::
+* getKeyAsString and key_as_string should be the same for terms aggregation on boolean field {pull}15393[#15393]
+
+Aliases::
+* make get alias expand to open and closed indices by default {pull}15954[#15954] (issue: {issue}14982[#14982])
+
+Allocation::
+* Simplify shard balancer interface {pull}17028[#17028] (issue: {issue}8954[#8954])
+* Remove DisableAllocationDecider {pull}13313[#13313]
+
+CAT API::
+* Add raw recovery progress to cat recovery API {pull}17064[#17064] (issue: {issue}17022[#17022])
+* Remove host from cat nodes API {pull}16656[#16656] (issues: {issue}12959[#12959], {issue}16575[#16575])
+* Using the accept header in the request instead of content-type in _cat API. {pull}14421[#14421] (issue: {issue}14195[#14195])
+
+CRUD::
+* Remove object notation for core types. {pull}15684[#15684] (issue: {issue}15388[#15388])
+
+Cache::
+* Refactor IndicesRequestCache to make it testable. {pull}16610[#16610]
+* Remove deprecated query cache settings {pull}15592[#15592]
+
+Core::
+* Bootstrap does not set system properties {pull}17088[#17088] (issues: {issue}16579[#16579], {issue}16791[#16791])
+* Add max number of processes check {pull}16919[#16919]
+* Add mlockall bootstrap check {pull}16909[#16909]
+* Remove es.useLinkedTransferQueue {pull}16786[#16786]
+* One log {pull}16703[#16703] (issue: {issue}16585[#16585])
+
+Engine::
+* Remove `index.compound_on_flush` setting and default to `true` {pull}15594[#15594] (issue: {issue}10778[#10778])
+
+Fielddata::
+* Remove "uninverted" and "binary" fielddata support for numeric and boolean fields. {pull}14082[#14082]
+
+Index APIs::
+* Remove `GET` option for /_forcemerge {pull}15223[#15223] (issue: {issue}15165[#15165])
+* Remove /_optimize REST API endpoint {pull}14226[#14226] (issue: {issue}13778[#13778])
+
+Internal::
+* Cli: Switch to jopt-simple {pull}17024[#17024] (issue: {issue}11564[#11564])
+* Replace ContextAndHeaders with a ThreadPool based ThreadLocal implementation {pull}15776[#15776]
+* Remove NodeBuilder {pull}15354[#15354]
+* Fix IndexSearcherWrapper interface to not depend on the EngineConfig {pull}14654[#14654]
+* Cleanup query parsing and remove IndexQueryParserService {pull}14452[#14452]
+* Fold IndexCacheModule into IndexModule {pull}14293[#14293]
+* Remove circular dependency between IndicesService and IndicesStore {pull}14285[#14285]
+* Remove guice injection from IndexStore and friends {pull}14279[#14279]
+* Simplify similarity module and friends {pull}13942[#13942]
+* Remove shard-level injector {pull}13881[#13881]
+* Refactor SearchRequest to be parsed on the coordinating node {pull}13859[#13859]
+* Remove support for pre 2.0 indices {pull}13799[#13799]
+
+Java API::
+* Remove the count api {pull}14166[#14166] (issue: {issue}13928[#13928])
+* IdsQueryBuilder to accept only non null ids and types {pull}13937[#13937]
+
+Mapping::
+* Change the field mapping index time boost into a query time boost. {pull}16900[#16900]
+* Deprecate string in favor of text/keyword. {pull}16877[#16877]
+* [Mapping] Several MappingService cleanups {pull}16133[#16133] (issue: {issue}15924[#15924])
+* [Mapping] Cleanup ParentFieldMapper: {pull}16045[#16045]
+* Remove the `format` option of the `_source` field. {pull}15398[#15398]
+* Remove transform {pull}13657[#13657] (issue: {issue}12674[#12674])
+
+Network::
+* Remove ability to disable Netty gathering writes {pull}16774[#16774] (issue: {issue}7811[#7811])
+
+Parent/Child::
+* Removed `total` score mode in favour for `sum` score mode. {pull}17174[#17174] (issues: {issue}13470[#13470], {issue}17083[#17083])
+* Several other parent/child cleanups {pull}13470[#13470]
+* Removed pre 2.x parent child implementation {pull}13376[#13376]
+
+Percolator::
+* Change the percolate api to not dynamically add fields to mapping {pull}16077[#16077] (issue: {issue}15751[#15751])
+
+Plugins::
+* Rename bin/plugin in bin/elasticsearch-plugin {pull}16454[#16454]
+* Change the inner structure of the plugins zip {pull}16453[#16453]
+* Remove multicast plugin {pull}16326[#16326] (issue: {issue}16310[#16310])
+* Plugins: Remove site plugins {pull}16038[#16038]
+* Don't use guice for QueryParsers {pull}15761[#15761]
+* Remove guice from the index level {pull}14518[#14518]
+* Simplify Analysis registration and configuration {pull}14355[#14355]
+* Replace IndicesLifecycle with a per-index IndexEventListener {pull}14217[#14217] (issue: {issue}13259[#13259])
+
+Query DSL::
+* Remove the MissingQueryBuilder which was deprecated in 2.2.0. {pull}15364[#15364] (issue: {issue}14112[#14112])
+* Remove NotQueryBuilder {pull}14204[#14204] (issue: {issue}13761[#13761])
+* Function score query: remove deprecated support for boost_factor {pull}13510[#13510]
+* Remove support for deprecated queries. {pull}13418[#13418] (issue: {issue}13326[#13326])
+
+REST::
+* Limit the accepted length of the _id {pull}16036[#16036] (issue: {issue}16034[#16034])
+
+Scripting::
+* Script settings {pull}16197[#16197]
+
+Search::
+* Remove some deprecations {pull}14331[#14331]
+* Remove search exists api {pull}13911[#13911] (issues: {issue}13682[#13682], {issue}13910[#13910])
+* Query refactoring: split parse phase into fromXContent and toQuery for all queries {pull}13788[#13788] (issue: {issue}10217[#10217])
+* Remove the scan and count search types. {pull}13310[#13310]
+
+Search Refactoring::
+* Remove deprecated parameter from field sort builder. {pull}16573[#16573] (issue: {issue}16127[#16127])
+* Remove support for query_binary and filter_binary {pull}14433[#14433] (issue: {issue}14308[#14308])
+* Validate query api: move query parsing to the coordinating node {pull}14384[#14384]
+* Remove "query" query and fix related parsing bugs {pull}14304[#14304] (issue: {issue}13326[#13326])
+
+Settings::
+* Prevent index level setting from being configured on a node level {pull}17144[#17144] (issue: {issue}16799[#16799])
+* Remove es.max-open-files flag {pull}16757[#16757] (issues: {issue}16506[#16506], {issue}483[#483])
+* Enforce node level limits if node is started in production env {pull}16733[#16733] (issue: {issue}16727[#16727])
+* Move remaining settings in NettyHttpServerTransport to the new infra {pull}16531[#16531]
+* Make settings validation strict {pull}16365[#16365]
+* Remove the ability to fsync on every operation and only schedule fsync task if really needed {pull}16257[#16257] (issue: {issue}16152[#16152])
+* Remove index.flush_on_close entirely {pull}15977[#15977]
+* Restore chunksize of 512kb on recovery and remove configurability {pull}15235[#15235] (issue: {issue}15161[#15161])
+* Remove ancient deprecated and alternative recovery settings {pull}15234[#15234]
+* Replace IndexSettings annotation with a full-fledged class {pull}14251[#14251]
+* Fix ping timeout settings inconsistencies {pull}13701[#13701] (issue: {issue}6579[#6579])
+
+Similarities::
+* Renames `default` similarity into `classic` {pull}15446[#15446] (issue: {issue}15102[#15102])
+
+Snapshot/Restore::
+* Fail closing or deleting indices during a full snapshot {pull}17021[#17021] (issue: {issue}16321[#16321])
+
+Stats::
+* Modify load average format {pull}15932[#15932] (issue: {issue}15907[#15907])
+* Reintroduce five-minute and fifteen-minute load averages on Linux {pull}15907[#15907] (issues: {issue}12049[#12049], {issue}14741[#14741])
+* Add system CPU percent to OS stats {pull}14741[#14741]
+
+Store::
+* Standardize state format type for global and index level metadata {pull}17123[#17123]
+
+Term Vectors::
+* Remove DFS support from TermVector API {pull}16452[#16452]
+* Term vector APIs should no longer update mappings {pull}16285[#16285]
+
+Translog::
+* Drop support for simple translog and hard-wire buffer to 8kb {pull}15574[#15574]
+* Simplify translog-based flush settings {pull}15573[#15573]
+
+Warmers::
+* Remove query warmers and the warmer API. {pull}15614[#15614] (issue: {issue}15607[#15607])
+
+
+
+[[deprecation-5.0.0-alpha1]]
+[float]
+=== Deprecations
+
+Plugin Mapper Attachment::
+* Deprecate mapper-attachments plugin {pull}16948[#16948] (issue: {issue}16910[#16910])
+
+Search::
+* Deprecate fuzzy query {pull}16211[#16211] (issues: {issue}15760[#15760], {issue}16121[#16121])
+
+
+
+[[feature-5.0.0-alpha1]]
+[float]
+=== New features
+
+Discovery::
+* Add two phased commit to Cluster State publishing {pull}13062[#13062]
+
+Ingest::
+* Merge feature/ingest branch into master branch {pull}16049[#16049] (issue: {issue}14049[#14049])
+
+Mapping::
+* Add a text field. {pull}16637[#16637]
+* Add a new `keyword` field. {pull}16589[#16589]
+
+Percolator::
+* index the query terms from the percolator query {pull}13646[#13646] (issue: {issue}12664[#12664])
+
+Plugin Ingest Attachment::
+* Ingest: Add attachment processor {pull}16490[#16490] (issue: {issue}16303[#16303])
+
+Plugin Mapper Attachment::
+* Migrate mapper attachments plugin to main repository {pull}14605[#14605]
+
+Plugin Repository HDFS::
+* HDFS Snapshot/Restore plugin {pull}15192[#15192] (issue: {issue}15191[#15191])
+
+Query DSL::
+* Adds a rewrite phase to queries on the shard level {pull}16870[#16870] (issue: {issue}9526[#9526])
+
+Reindex API::
+* Merge reindex to master {pull}16861[#16861]
+
+Scripting::
+* Exceptions and Infinite Loop Checking {pull}15936[#15936]
+* Added a new scripting language (PlanA) {pull}15136[#15136] (issue: {issue}13084[#13084])
+
+Search::
+* Add `search_after` parameter in the SearchAPI {pull}16125[#16125] (issue: {issue}8192[#8192])
+
+Settings::
+* Add infrastructure to transactionally apply and reset dynamic settings {pull}15278[#15278]
+
+Stats::
+* API for listing index file sizes {pull}16661[#16661] (issue: {issue}16131[#16131])
+
+Suggesters::
+* Add document-oriented completion suggester {pull}14410[#14410] (issue: {issue}10746[#10746])
+
+Task Manager::
+* Add task cancellation mechanism {pull}16320[#16320]
+* Make the Task object available to the action caller {pull}16033[#16033]
+* Task Management: Add framework for registering and communicating with tasks {pull}15347[#15347] (issue: {issue}15117[#15117])
+
+
+
+[[enhancement-5.0.0-alpha1]]
+[float]
+=== Enhancements
+
+Aggregations::
+* Add tests and documentation for using `time_zone` in date range aggregation {pull}16955[#16955] (issue: {issue}10130[#10130])
+* Refactoring of Aggregations {pull}14136[#14136]
+
+Allocation::
+* Write shard state metadata as soon as shard is created / initializing {pull}16625[#16625] (issue: {issue}14739[#14739])
+* Reuse existing allocation id for primary shard allocation {pull}16530[#16530] (issue: {issue}14739[#14739])
+* Remove version in ShardRouting (now obsolete) {pull}16243[#16243] (issue: {issue}14739[#14739])
+* Prefer nodes that previously held primary shard for primary shard allocation {pull}16096[#16096] (issue: {issue}14739[#14739])
+* Extend reroute with an option to force assign stale primary shard copies {pull}15708[#15708] (issue: {issue}14739[#14739])
+* Allocate primary shards based on allocation IDs {pull}15281[#15281] (issue: {issue}14739[#14739])
+* Persist currently started allocation IDs to index metadata {pull}14964[#14964] (issue: {issue}14739[#14739])
+* Use ObjectParser to parse AllocationID {pull}14962[#14962] (issue: {issue}14831[#14831])
+* Persist allocation ID with shard state metadata on nodes {pull}14831[#14831] (issue: {issue}14739[#14739])
+
+CAT API::
+* Expose http address in cat/nodes {pull}16770[#16770]
+* [cat/recovery] Make recovery time a TimeValue() {pull}16743[#16743] (issue: {issue}9209[#9209])
+* :CAT API: remove space at the end of a line {pull}15250[#15250] (issue: {issue}9464[#9464])
+
+CRUD::
+* CRUD: Allow to get and set ttl as a time value/string {pull}15047[#15047]
+
+Cache::
+* Enable the indices request cache by default {pull}17162[#17162] (issues: {issue}16870[#16870], {issue}17134[#17134])
+
+Cluster::
+* Resolve index names to Index instances early {pull}17048[#17048]
+* Remove DiscoveryNode#shouldConnectTo method {pull}16898[#16898] (issue: {issue}16815[#16815])
+* Fail demoted primary shards and retry request {pull}16415[#16415] (issue: {issue}14252[#14252])
+* Illegal shard failure requests {pull}16275[#16275]
+* Shard failure requests for non-existent shards {pull}16089[#16089] (issue: {issue}14252[#14252])
+* Add handling of channel failures when starting a shard {pull}16041[#16041] (issue: {issue}15895[#15895])
+* Wait for new master when failing shard {pull}15748[#15748] (issue: {issue}14252[#14252])
+* Master should wait on cluster state publication when failing a shard {pull}15468[#15468] (issue: {issue}14252[#14252])
+* Split cluster state update tasks into roles {pull}14899[#14899] (issue: {issue}13627[#13627])
+* Add timeout mechanism for sending shard failures {pull}14707[#14707] (issue: {issue}14252[#14252])
+* Add listener mechanism for failures to send shard failed {pull}14295[#14295] (issue: {issue}14252[#14252])
+
+Core::
+* Use index UUID to lookup indices on IndicesService {pull}17001[#17001]
+* Add -XX+AlwaysPreTouch JVM flag {pull}16937[#16937]
+* Use and test relative time in TransportBulkAction {pull}16916[#16916]
+* Bump Elasticsearch version to 5.0.0-alpha1-SNAPSHOT {pull}16862[#16862]
+* Assert that we can write in all data-path on startup {pull}16745[#16745]
+* Add G1GC check on startup {pull}16737[#16737] (issue: {issue}10740[#10740])
+* Shards with heavy indexing should get more of the indexing buffer {pull}14121[#14121]
+* Remove and ban ImmutableMap {pull}13939[#13939] (issue: {issue}13224[#13224])
+* Finish banning ImmutableSet {pull}13820[#13820] (issue: {issue}13224[#13224])
+* Removes and bans ImmutableSet {pull}13754[#13754] (issue: {issue}13224[#13224])
+* Remove and ban ImmutableMap#entrySet {pull}13724[#13724]
+* Forbid ForwardingSet {pull}13720[#13720] (issue: {issue}13224[#13224])
+
+Discovery::
+* Add a dedicate queue for incoming ClusterStates {pull}13303[#13303] (issue: {issue}13062[#13062])
+
+Engine::
+* Remove writeLockTimeout from InternalEngine {pull}16930[#16930]
+* Don't guard IndexShard#refresh calls by a check to isRefreshNeeded {pull}16118[#16118]
+* Never call a listerner under lock in InternalEngine {pull}15786[#15786]
+* Use System.nanoTime() to initialize Engine.lastWriteNanos {pull}14321[#14321]
+* Flush big merges automatically if shard is inactive {pull}14275[#14275]
+* Remove Engine.Create {pull}13955[#13955]
+* Remove the disabled autogenerated id optimization from InternalEngine {pull}13857[#13857]
+
+Exceptions::
+* Fix typos in exception/assert/log messages in core module. {pull}16649[#16649]
+* Add field names to several mapping errors {pull}16508[#16508] (issue: {issue}16378[#16378])
+* Add serialization support for more important IOExceptions {pull}15766[#15766]
+* Adds exception objects to log messages. {pull}14827[#14827] (issue: {issue}10021[#10021])
+* Remove reflection hacks from ElasticsearchException {pull}13796[#13796]
+* Rename QueryParsingException to a more generic ParsingException {pull}13631[#13631]
+* Add *Exception(Throwable cause) constructors/ call where appropriate {pull}13544[#13544] (issue: {issue}10021[#10021])
+
+Geo::
+* Fix a potential parsing problem in GeoDistanceSortParser {pull}17111[#17111]
+* Geo: Add validation of shapes to ShapeBuilders {pull}15551[#15551] (issue: {issue}14416[#14416])
+* Make remaining ShapeBuilders implement Writeable {pull}15010[#15010] (issue: {issue}14416[#14416])
+* Geo: Remove internal `translated` flag from LineStringBuilder {pull}14969[#14969]
+* Make PointBuilder, CircleBuilder & EnvelopeBuilder implement Writable  {pull}14933[#14933] (issue: {issue}14416[#14416])
+* Merging BaseLineString and BasePolygonBuilder with subclass {pull}14887[#14887] (issue: {issue}14482[#14482])
+* Moving static factory methods to ShapeBuilders {pull}14529[#14529]
+* Remove InternalLineStringBuilder and InternalPolygonBuilder {pull}14482[#14482] (issue: {issue}14416[#14416])
+
+Highlighting::
+* Joint parsing of common global Hightlighter and subfield parameters {pull}15368[#15368] (issue: {issue}15285[#15285])
+*  Add fromXContent method to HighlightBuilder {pull}15157[#15157]
+
+Ingest::
+* Added ingest statistics to node stats API {pull}16915[#16915]
+* Add `ingest_took` to bulk response {pull}16876[#16876]
+* Add ingest info to node info API, which contains a list of available processors {pull}16865[#16865]
+* Use diffs for ingest metadata in cluster state {pull}16847[#16847]
+* hide null-valued metadata fields from WriteableIngestDocument#toXContent {pull}16557[#16557]
+* Ingest: use bulk thread pool for bulk request processing (was index before) {pull}16539[#16539] (issue: {issue}16503[#16503])
+* Add foreach processor {pull}16432[#16432]
+* revert PipelineFactoryError handling with throwing ElasticsearchParseException in ingest pipeline creation {pull}16355[#16355]
+* Add processor tags to on_failure metadata in ingest pipeline {pull}16324[#16324] (issue: {issue}16202[#16202])
+* catch processor/pipeline factory exceptions and return structured error responses {pull}16276[#16276] (issue: {issue}16010[#16010])
+* Ingest: move get/put/delete pipeline methods to ClusterAdminClient {pull}16242[#16242]
+* Geoip processor: remove redundant latitude and longitude fields and make location an object with lat and lon subfields {pull}16173[#16173]
+
+Internal::
+* Support scheduled commands in current context {pull}17077[#17077]
+* Thread limits {pull}17003[#17003]
+* Remove leniency from segments info integrity checks {pull}16985[#16985] (issue: {issue}16973[#16973])
+* Rename SearchServiceTransportAction to SearchTransportService {pull}16880[#16880]
+* Decouple the TransportService and ClusterService {pull}16872[#16872] (issue: {issue}16788[#16788])
+* Refactor bootstrap checks {pull}16844[#16844] (issues: {issue}16733[#16733], {issue}16835[#16835])
+* Add LifecycleRunnable {pull}16752[#16752]
+* Hot inlined methods in your area {pull}16725[#16725]
+* Move IndicesQueryCache and IndicesRequestCache into IndicesService {pull}16603[#16603]
+* Forbid use of java.security.MessageDigest#clone() {pull}16543[#16543] (issue: {issue}16479[#16479])
+* Make IndicesWarmer a private class of IndexService {pull}16470[#16470]
+* Simplify IndicesFieldDataCache and detach from guice {pull}16469[#16469]
+* Uppercase ells ('L') in long literals {pull}16329[#16329] (issue: {issue}16279[#16279])
+* ShardId equality and hash code inconsistency {pull}16319[#16319] (issue: {issue}16217[#16217])
+* Ensure all resources are closed on Node#close() {pull}16316[#16316] (issue: {issue}13685[#13685])
+* Make index uuid available in Index, ShardRouting & ShardId {pull}16217[#16217]
+* Move RefreshTask into IndexService and use since task per index {pull}15933[#15933]
+* Make IndexingMemoryController private to IndicesService {pull}15877[#15877]
+* Cleanup IndexingOperationListeners infrastructure {pull}15875[#15875]
+* Remove and forbid use of j.u.c.ThreadLocalRandom {pull}15862[#15862] (issue: {issue}15294[#15294])
+* Fix IntelliJ query builder type inference issues {pull}15429[#15429]
+* Remove and forbid use of Collections#shuffle(List) and Random#<init>() {pull}15299[#15299] (issue: {issue}15287[#15287])
+* Remove and forbid use of the type-unsafe empty Collections fields {pull}15187[#15187]
+* Move IndicesService.canDeleteShardContent to use IndexSettings {pull}15150[#15150] (issue: {issue}15059[#15059])
+* Simplify MonitorService construction and detach from guice {pull}15035[#15035]
+* Use Supplier for StreamInput#readOptionalStreamable {pull}14806[#14806]
+* Add variable-length long encoding {pull}14780[#14780]
+* Extend usage of IndexSetting class {pull}14731[#14731] (issue: {issue}14251[#14251])
+* Fold SimilarityModule into IndexModule {pull}14284[#14284]
+* Move to lucene BoostQuery {pull}14264[#14264]
+* Use built-in method for computing hash code of longs {pull}14213[#14213]
+* Refactor ShardFailure listener infrastructure {pull}14206[#14206]
+* Add methods for variable-length encoding integral arrays {pull}14087[#14087]
+* Fold IndexAliasesService into IndexService {pull}14044[#14044]
+* Remove unneeded Module abstractions {pull}13944[#13944]
+* Query refactoring: simplify IndexQueryParserService parse methods {pull}13938[#13938] (issue: {issue}13859[#13859])
+* Remove and forbid use of com.google.common.collect.Iterators {pull}13916[#13916] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.collect.ImmutableCollection {pull}13909[#13909] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.io.Resources {pull}13908[#13908] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.hash.* {pull}13907[#13907] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.net.InetAddresses {pull}13905[#13905] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.collect.EvictingQueue {pull}13903[#13903] (issue: {issue}13224[#13224])
+* Replace Guava cache with simple concurrent LRU cache {pull}13879[#13879]
+* Remove ClusterSerivce and IndexSettingsService dependency from IndexShard {pull}13853[#13853]
+* Start making RecoverySourceHandler unittestable {pull}13840[#13840]
+* Remove IndexService dep. from IndexShard {pull}13797[#13797]
+* Remove ES internal deletion policies in favour of Lucenes implementations {pull}13794[#13794]
+* Move ShardTermVectorService to be on indices level as TermVectorService {pull}13786[#13786]
+* Move ShardPercolateService creation into IndexShard {pull}13777[#13777]
+* Remove `ExpressionScriptCompilationException` and `ExpressionScriptExecutionException` {pull}13742[#13742]
+* Reduced the number of ClusterStateUpdateTask variants {pull}13735[#13735]
+* Add a BaseParser helper for stream parsing {pull}13615[#13615]
+* Remove and forbid use of com.google.common.primitives.Ints {pull}13596[#13596] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.math.LongMath {pull}13575[#13575] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.base.Joiner {pull}13572[#13572] (issue: {issue}13224[#13224])
+* Replace and ban next batch of Guava classes {pull}13562[#13562] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.collect.Iterables {pull}13559[#13559] (issue: {issue}13224[#13224])
+* Replace LoadingCache usage with a simple ConcurrentHashMap {pull}13552[#13552] (issue: {issue}13224[#13224])
+* Use Supplier instead of Reflection {pull}13545[#13545]
+* Remove and forbid use of com.google.common.base.Preconditions {pull}13540[#13540] (issue: {issue}13224[#13224])
+* Remove and forbid use of guava Function, Charsets, Collections2 {pull}13533[#13533] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.collect.ImmutableSortedMap {pull}13525[#13525] (issue: {issue}13224[#13224])
+* Remove and forbid use of several com.google.common.util. classes {pull}13524[#13524] (issue: {issue}13224[#13224])
+* Cleanup SearchRequest & SearchRequestBuilder {pull}13518[#13518]
+* Remove and forbid use of com.google.common.collect.Queues {pull}13498[#13498] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.base.Preconditions#checkNotNull {pull}13493[#13493] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.collect.Sets {pull}13463[#13463] (issue: {issue}13224[#13224])
+* Remove and forbid use of com.google.common.collect.Maps {pull}13438[#13438] (issue: {issue}13224[#13224])
+* Remove use of underscore as an identifier {pull}13353[#13353]
+* Remove and forbid the use of com.google.common.base.Predicate(s)? {pull}13349[#13349] (issues: {issue}13224[#13224], {issue}13314[#13314])
+* This commit removes com.google.common.io {pull}13302[#13302] (issue: {issue}13224[#13224])
+
+Java API::
+* Remove copy constructors from request classes and TransportMessage type {pull}16640[#16640] (issue: {issue}15776[#15776])
+
+Mapping::
+* Remove friction from the mapping changes in 5.0. {pull}16991[#16991]
+* Rework norms parameters for 5.0. {pull}16987[#16987]
+* Moved dynamic field handling in doc parsing to end of parsing {pull}16798[#16798]
+* Remove the MapperBuilders utility class. {pull}16609[#16609]
+* Make the `index` property a boolean. {pull}16161[#16161]
+* Remove the ability to enable doc values with the `fielddata.format` setting. {pull}16147[#16147]
+* Be stricter about parsing boolean values in mappings. {pull}16146[#16146]
+* Fix default doc values to be enabled when a field is not indexed. {pull}16141[#16141]
+* Dynamically map floating-point numbers as floats instead of doubles. {pull}15319[#15319] (issue: {issue}13851[#13851])
+* Simplify MetaDataMappingService. {pull}15217[#15217]
+* Remove MergeMappingException. {pull}15177[#15177]
+
+NOT CLASSIFIED::
+* Use a seed node to form multi-node cluster in integ tests {pull}17078[#17078]
+
+Packaging::
+* Fail early on JDK with compiler bug {pull}16418[#16418] (issues: {issue}16097[#16097], {issue}16362[#16362])
+* Make security non-optional {pull}16176[#16176]
+* Remove RuntimePermission("accessDeclaredMembers") {pull}15378[#15378]
+* Remove Guava as a dependency {pull}14055[#14055] (issue: {issue}13224[#13224])
+* Remove Guava as a dependency {pull}14054[#14054] (issue: {issue}13224[#13224])
+
+Plugin Ingest Attachment::
+* Minor attachment processor improvements {pull}16574[#16574]
+
+Plugin Lang Painless::
+* Make Painless a Module {pull}16755[#16755]
+* Minor Clean up {pull}16457[#16457]
+
+Plugin Mapper Attachment::
+* minor attachments cleanups: IDE test support and EPUB format {pull}14626[#14626]
+
+Plugin Repository Azure::
+* Support global `repositories.azure.` settings {pull}15141[#15141] (issue: {issue}13776[#13776])
+* Add timeout settings (default to 5 minutes) {pull}15080[#15080] (issue: {issue}14277[#14277])
+
+Plugin Repository HDFS::
+* merge current hdfs improvements to master {pull}15588[#15588]
+
+Plugin Repository S3::
+* Add support for proxy authentication for s3 and ec2 {pull}15293[#15293] (issue: {issue}15268[#15268])
+
+Plugins::
+* CliTool: Cleanup and document Terminal {pull}16443[#16443]
+* Plugin cli: Improve maven coordinates detection {pull}16384[#16384] (issue: {issue}16376[#16376])
+* Enforce plugin zip does not contain zip entries outside of the plugin dir {pull}16361[#16361]
+* CliTool: Allow unexpected exceptions to propagate {pull}16359[#16359]
+* Reduce complexity of plugin cli {pull}16336[#16336]
+* Remove Plugin.onIndexService. {pull}15029[#15029] (issue: {issue}14896[#14896])
+* Open up QueryCache and SearcherWrapper extension points {pull}14303[#14303]
+
+Query DSL::
+* Function Score Query: make parsing stricter {pull}16617[#16617] (issue: {issue}16583[#16583])
+* Parsers should throw exception on unknown objects {pull}14255[#14255] (issue: {issue}10974[#10974])
+* UNICODE_CHARACTER_CLASS fix {pull}11598[#11598] (issue: {issue}10146[#10146])
+
+Query Refactoring::
+* Add infrastructure to rewrite query builders {pull}16599[#16599]
+* Switch geo validation to enum {pull}13672[#13672] (issue: {issue}13608[#13608])
+
+REST::
+* More robust handling of CORS HTTP Access Control {pull}16092[#16092]
+* Add option to exclude based on paths in XContent {pull}16017[#16017]
+
+Recovery::
+* Relocation source should be marked as relocating before starting recovery to primary relocation target {pull}16500[#16500]
+* Operation counter for IndexShard {pull}15956[#15956] (issue: {issue}15900[#15900])
+* Primary relocation handoff {pull}15900[#15900] (issue: {issue}15532[#15532])
+* Remove recovery threadpools and throttle outgoing recoveries on the master {pull}15372[#15372]
+* Refactor StoreRecoveryService to be a simple package private util class {pull}13766[#13766]
+
+Reindex API::
+* Add ingest pipeline support to reindex {pull}16932[#16932]
+
+Scripting::
+* Remove Extra String Concat Token {pull}16382[#16382]
+* Skipping hidden files compilation for script service {pull}16286[#16286] (issue: {issue}15269[#15269])
+* Rename Plan A to Painless {pull}16245[#16245]
+* Add plumbing for script compile-time parameters {pull}15464[#15464]
+* Factor mustache -> modules/lang-mustache {pull}15328[#15328]
+
+Search::
+* Store _all payloads on 1 byte instead of 4. {pull}16899[#16899]
+* Refuse to load fields from _source when using the `fields` option and support wildcards.  {pull}15017[#15017] (issues: {issue}10783[#10783], {issue}14489[#14489])
+* Add response into ClearScrollResponse {pull}13835[#13835] (issue: {issue}13817[#13817])
+* Shuffle shards for _only_nodes + support multiple specifications like cluster API  {pull}12575[#12575] (issues: {issue}12546[#12546], {issue}12700[#12700])
+
+Search Refactoring::
+* Move sort `order` field up into SortBuilder {pull}17035[#17035]
+* Moves SortParser:parse(...) to only require QueryShardContext {pull}16999[#16999] (issue: {issue}15178[#15178])
+* Change internal representation of suggesters  {pull}16873[#16873]
+* Fixes serialisation of Ranges {pull}16674[#16674]
+* Make GeoDistanceSortBuilder serializable, 2nd try {pull}16572[#16572] (issues: {issue}15178[#15178], {issue}16151[#16151])
+* Move missing() from SortBuilder interface to class {pull}16225[#16225] (issues: {issue}15178[#15178], {issue}16151[#16151])
+* Remove deprecated parameters from ScriptSortBuilder {pull}16153[#16153] (issue: {issue}15178[#15178])
+* Refactor GeoSortBuilder {pull}16151[#16151] (issue: {issue}15178[#15178])
+* Refactor FieldSortBuilder {pull}16127[#16127] (issue: {issue}15178[#15178])
+* Make sort order enum writable. {pull}16124[#16124] (issue: {issue}15178[#15178])
+* Make DistanceUnit writable. {pull}16122[#16122] (issue: {issue}15178[#15178])
+* RescoreBuilder: Add parsing and creating of RescoreSearchContext {pull}16014[#16014] (issue: {issue}15559[#15559])
+* Make RescoreBuilder and nested QueryRescorer Writable {pull}15953[#15953] (issue: {issue}15559[#15559])
+* Use HighlightBuilder in SearchSourceBuilder {pull}15376[#15376] (issue: {issue}15044[#15044])
+* Enable HighlightBuilder to create SearchContextHighlight {pull}15324[#15324]
+* Explain api: move query parsing to the coordinating node {pull}14270[#14270]
+* Switch query parsers to use ParseField  {pull}14249[#14249] (issue: {issue}8964[#8964])
+
+Settings::
+* Settings with complex matchers should not overlap {pull}16754[#16754]
+* Moves GCE settings to the new infra {pull}16722[#16722] (issue: {issue}16720[#16720])
+* Add filtering support within Setting class {pull}16629[#16629] (issue: {issue}16598[#16598])
+* Migrate AWS settings to new settings infrastructure {pull}16602[#16602] (issue: {issue}16293[#16293])
+* Remove `gateway.initial_meta` and always rely on min master nodes {pull}16446[#16446]
+* Rewrite SettingsFilter to be immutable {pull}16425[#16425]
+* Simplify azure settings {pull}16363[#16363]
+* Convert PageCacheRecycler settings {pull}16341[#16341]
+* Monitor settings {pull}16313[#16313]
+* Cut over tribe node settings to new settings infra {pull}16311[#16311]
+* Convert multcast plugin settings to the new infra {pull}16295[#16295]
+* Convert `request.headers.*` to the new settings infra {pull}16292[#16292]
+* Migrate Azure settings to new settings infrastructure {pull}16291[#16291]
+* Validate logger settings and allow them to be reset via API {pull}16289[#16289]
+* Switch NodeEnvironment's settings to new settings {pull}16273[#16273]
+* Simplify AutoCreateIndex and add more tests {pull}16270[#16270]
+* Convert several pending settings {pull}16269[#16269]
+* Migrate query caching settings to the new settings infra. {pull}16267[#16267]
+* Convert `action.auto_create_index` and `action.master.force_local` to the new settings infra {pull}16263[#16263]
+* Convert `cluster.routing.allocation.type` and `processors` to the new settings infra. {pull}16238[#16238]
+* Validate tribe node settings on startup {pull}16237[#16237]
+* Move node.client, node.data, node.master, node.local and node.mode to new settings infra {pull}16230[#16230]
+* Moved http settings to the new settings infrastructure {pull}16188[#16188]
+* Migrate network service to the new infra {pull}16187[#16187]
+* Convert client.transport settings to new infra {pull}16183[#16183]
+* Move discovery.* settings to new Setting infrastructure {pull}16182[#16182]
+* Change over to o.e.common.settings.Setting for http settings {pull}16181[#16181]
+* Convert "path.*" and "pidfile" to new settings infra {pull}16180[#16180]
+* Migrate repository settings to the new settings API {pull}16178[#16178]
+* Convert "indices.*" settings to new infra. {pull}16177[#16177]
+* Migrate gateway settings to the new settings API. {pull}16175[#16175]
+* Convert several node and test level settings {pull}16172[#16172]
+* Run Metadata upgrade tool on every version {pull}16168[#16168]
+* Check for invalid index settings on metadata upgrade {pull}16156[#16156]
+* Validate the settings key if it's simple chars separated by `.` {pull}16120[#16120]
+* Validate known global settings on startup {pull}16091[#16091]
+* Cut over all index scope settings to the new setting infrastrucuture {pull}16054[#16054] (issues: {issue}12790[#12790], {issue}12854[#12854], {issue}16032[#16032], {issue}6732[#6732])
+* Remove updatability of `index.flush_on_close` {pull}15964[#15964] (issue: {issue}15955[#15955])
+* Move all dynamic settings and their config classes to the index level {pull}15955[#15955] (issue: {issue}6732[#6732])
+* Always require units for bytes and time settings {pull}15948[#15948] (issue: {issue}11437[#11437])
+* Make MetaData parsing less lenient. {pull}15828[#15828]
+* Move async translog sync logic into IndexService {pull}15584[#15584]
+* Remove `index.merge.scheduler.notify_on_failure` and default to `true` {pull}15572[#15572] (issue: {issue}15570[#15570])
+* Remove cache concurrency level settings that no longer apply {pull}14210[#14210] (issues: {issue}13224[#13224], {issue}13717[#13717], {issue}7836[#7836])
+
+Similarities::
+* Defining a global default similarity {pull}16682[#16682] (issue: {issue}16594[#16594])
+
+Snapshot/Restore::
+* Remove AbstractLegacyBlobContainer {pull}14650[#14650] (issue: {issue}13434[#13434])
+
+Stats::
+* Normalize unavailable load average {pull}16061[#16061] (issues: {issue}12049[#12049], {issue}14741[#14741], {issue}15907[#15907], {issue}15932[#15932], {issue}15934[#15934])
+* Add load averages to OS stats on FreeBSD {pull}15934[#15934] (issue: {issue}15917[#15917])
+* Expose pending cluster state queue size in node stats {pull}14040[#14040] (issue: {issue}13610[#13610])
+
+Store::
+* Remove support for legacy checksums {pull}16931[#16931]
+* Rename index folder to index_uuid {pull}16442[#16442] (issues: {issue}13264[#13264], {issue}13265[#13265], {issue}14512[#14512], {issue}14932[#14932], {issue}15853[#15853])
+
+Suggesters::
+* Refactoring of Suggestions {pull}17096[#17096] (issue: {issue}10217[#10217])
+
+Task Manager::
+* Add start time and duration to tasks {pull}16829[#16829]
+* Combine node name and task id into single string task id {pull}16744[#16744]
+* Add task status {pull}16356[#16356] (issue: {issue}16344[#16344])
+* Extend tracking of parent tasks to master node, replication and broadcast actions {pull}15931[#15931]
+
+Translog::
+* Remove ChannelReference and simplify Views {pull}15898[#15898]
+* Simplify TranslogWriter to always write to a stream {pull}15771[#15771]
+* Remove TranslogService and fold it into synchronous IndexShard API {pull}13707[#13707]
+
+
+
+[[bug-5.0.0-alpha1]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Correct typo in class name of StatsAggregator {pull}15264[#15264] (issue: {issue}14730[#14730])
+
+Allocation::
+* Replica shards must be failed before primary shards {pull}15686[#15686]
+
+CRUD::
+* Prevent TransportReplicationAction to route request based on stale local routing table {pull}16274[#16274] (issues: {issue}12573[#12573], {issue}12574[#12574])
+* Resolves the conflict between alias routing and parent routing by applying the alias routing and ignoring the parent routing. {pull}15371[#15371] (issue: {issue}3068[#3068])
+
+Cluster::
+* Shard state action channel exceptions {pull}16057[#16057] (issue: {issue}15748[#15748])
+
+Geo::
+* Fix multi-field support for GeoPoint types {pull}15702[#15702] (issue: {issue}15701[#15701])
+* Enforce distance in distance query is > 0 {pull}15135[#15135]
+
+Ingest::
+* The IngestDocument copy constructor should make a deep copy {pull}16248[#16248] (issue: {issue}16246[#16246])
+
+Internal::
+* Enable unmap hack for java 9 {pull}16986[#16986] (issue: {issue}1[#1])
+* Fix issues with failed cache loads {pull}14315[#14315]
+* Allow parser to move on the START_OBJECT token when parsing search source {pull}14145[#14145]
+* Ensure searcher is release if wrapping fails {pull}14107[#14107]
+* Avoid deadlocks in Cache#computeIfAbsent {pull}14091[#14091] (issue: {issue}14090[#14090])
+
+Java API::
+* Fix potential NPE in SearchSourceBuilder {pull}16905[#16905] (issue: {issue}16902[#16902])
+
+Mapping::
+* Fix dynamic mapper when its parent already has an update {pull}17065[#17065]
+* Fix copy_to when the target is a dynamic object field. {pull}15216[#15216] (issues: {issue}111237[#111237], {issue}11237[#11237])
+* Preserve existing mappings on batch mapping updates {pull}15130[#15130] (issues: {issue}14899[#14899], {issue}15129[#15129])
+
+Packaging::
+* Do not pass double-dash arguments on startup {pull}17087[#17087] (issue: {issue}17084[#17084])
+
+Plugin Store SMB::
+* Fix calling ensureOpen() on the wrong directory (master forwardport) {pull}16395[#16395] (issue: {issue}16383[#16383])
+
+Plugins::
+* CliTool: Messages printed in Terminal should have percent char escaped {pull}16367[#16367]
+
+Query DSL::
+* `constant_score` query should throw error on more than one filter {pull}17135[#17135] (issue: {issue}17126[#17126])
+* Single IPv4 addresses in IP field term queries {pull}16068[#16068] (issue: {issue}16058[#16058])
+* Make strategy optional in GeoShapeQueryBuilder readFrom and writeTo {pull}13963[#13963]
+
+Query Refactoring::
+* Query refactoring: set has_parent & has_child types context properly {pull}13863[#13863]
+* Make sure equivalent geohashCellQueries are equal after toQuery called {pull}13792[#13792]
+
+Recovery::
+* Prevent interruption while store checks lucene files for consistency {pull}16308[#16308]
+* Mark shard as recovering on the cluster state thread {pull}14276[#14276] (issues: {issue}13766[#13766], {issue}14115[#14115])
+
+Search::
+* Fix for search after {pull}16271[#16271]
+* Do not be lenient when parsing CIDRs {pull}14874[#14874] (issue: {issue}14862[#14862])
+
+Settings::
+* Register bootstrap settings {pull}16513[#16513]
+* Add settings filtering to node info requests {pull}16445[#16445]
+* Ban write access to system properties {pull}14914[#14914]
+
+Translog::
+* Mark shard active during recovery; push settings after engine finally inits {pull}16250[#16250] (issues: {issue}14121[#14121], {issue}16209[#16209])
+
+
+
+[[upgrade-5.0.0-alpha1]]
+[float]
+=== Upgrades
+
+Core::
+* Upgrade to lucene-6.0.0-f0aa4fc. {pull}17075[#17075]
+* upgrade to lucene 6.0.0-snapshot-bea235f {pull}16964[#16964]
+* Upgrade to Jackson 2.7.1 {pull}16801[#16801] (issue: {issue}16294[#16294])
+
+Ingest::
+* Update MaxMind geoip2 version to 2.6 {pull}16837[#16837] (issue: {issue}16801[#16801])
+
+Internal::
+* Bump master (3.0-snapshot) to java 8 {pull}13314[#13314]
+
+Search Templates::
+* Update mustache.java to version 0.9.1 {pull}14053[#14053] (issue: {issue}13224[#13224])
+
+
diff --git a/docs/reference/search/percolate.asciidoc b/docs/reference/search/percolate.asciidoc
index bb4040770eb..700c4ed392c 100644
--- a/docs/reference/search/percolate.asciidoc
+++ b/docs/reference/search/percolate.asciidoc
@@ -1,513 +1,8 @@
 [[search-percolate]]
 == Percolator
 
-added[5.0.0,Percolator queries modifications aren't visible immediately and a refresh is required]
+deprecated[5.0.0,Percolate and multi percolate APIs are deprecated and have been replaced by the new <<query-dsl-percolator-query,`percolator` query>>]
 
-added[5.0.0,Percolate api by defaults limits the number of matches to `10` whereas before this wasn't set]
+added[5.0.0,Percolator query modifications only become visible after a refresh has occurred. Previously, they became visible immediately]
 
-added[5.0.0,For indices created on or after version 5.0.0 the percolator automatically indexes the query terms with the percolator queries this allows the percolator to percolate documents quicker. It is advisable to reindex any pre 5.0.0 indices to take advantage of this new optimization]
-
-Traditionally you design documents based on your data, store them into an index, and then define queries via the search API
-in order to retrieve these documents. The percolator works in the opposite direction. First you store queries into an
-index and then, via the percolate API, you define documents in order to retrieve these queries.
-
-The reason that queries can be stored comes from the fact that in Elasticsearch both documents and queries are defined in
-JSON. This allows you to embed queries into documents via the index API. Elasticsearch can extract the query from a
-document and make it available to the percolate API. Since documents are also defined as JSON, you can define a document
-in a request to the percolate API.
-
-[IMPORTANT]
-=====================================
-
-Fields referred to in a percolator query must *already* exist in the mapping
-associated with the index used for percolation. In order to make sure these fields exist,
-add or update a mapping via the <<indices-create-index,create index>> or <<indices-put-mapping,put mapping>> APIs.
-
-=====================================
-
-[float]
-=== Sample Usage
-
-Create an index with a mapping for the field `message`:
-
-[source,js]
---------------------------------------------------
-curl -XPUT 'localhost:9200/my-index' -d '{
-  "mappings": {
-    "my-type": {
-      "properties": {
-        "message": {
-          "type": "string"
-        }
-      }
-    }
-  }
-}'
---------------------------------------------------
-
-Register a query in the percolator:
-
-[source,js]
---------------------------------------------------
-curl -XPUT 'localhost:9200/my-index/.percolator/1' -d '{
-    "query" : {
-        "match" : {
-            "message" : "bonsai tree"
-        }
-    }
-}'
---------------------------------------------------
-
-Match a document to the registered percolator queries:
-
-[source,js]
---------------------------------------------------
-curl -XGET 'localhost:9200/my-index/my-type/_percolate' -d '{
-    "doc" : {
-        "message" : "A new bonsai tree in the office"
-    }
-}'
---------------------------------------------------
-
-The above request will yield the following response:
-
-[source,js]
---------------------------------------------------
-{
-    "took" : 19,
-    "_shards" : {
-        "total" : 5,
-        "successful" : 5,
-        "failed" : 0
-    },
-    "total" : 1,
-    "matches" : [ <1>
-    	{
-    	  "_index" : "my-index",
-    	  "_id" : "1"
-    	}
-    ]
-}
---------------------------------------------------
-
-<1> The percolate query with id `1` matches our document.
-
-[float]
-=== Indexing Percolator Queries
-
-Percolate queries are stored as documents in a specific format and in an arbitrary index under a reserved type with the
-name `.percolator`. The query itself is placed as is in a JSON object under the top level field `query`.
-
-[source,js]
---------------------------------------------------
-{
-    "query" : {
-		"match" : {
-			"field" : "value"
-		}
-	}
-}
---------------------------------------------------
-
-Since this is just an ordinary document, any field can be added to this document. This can be useful later on to only
-percolate documents by specific queries.
-
-[source,js]
---------------------------------------------------
-{
-	"query" : {
-		"match" : {
-			"field" : "value"
-		}
-	},
-	"priority" : "high"
-}
---------------------------------------------------
-
-On top of this, also a mapping type can be associated with this query. This allows to control how certain queries
-like range queries, shape filters, and other query & filters that rely on mapping settings get constructed. This is
-important since the percolate queries are indexed into the `.percolator` type, and the queries / filters that rely on
-mapping settings would yield unexpected behaviour. Note: By default, field names do get resolved in a smart manner,
-but in certain cases with multiple types this can lead to unexpected behavior, so being explicit about it will help.
-
-[source,js]
---------------------------------------------------
-{
-	"query" : {
-		"range" : {
-			"created_at" : {
-				"gte" : "2010-01-01T00:00:00",
-				"lte" : "2011-01-01T00:00:00"
-			}
-		}
-	},
-	"type" : "tweet",
-	"priority" : "high"
-}
---------------------------------------------------
-
-In the above example the range query really gets parsed into a Lucene numeric range query, based on the settings for
-the field `created_at` in the type `tweet`.
-
-Just as with any other type, the `.percolator` type has a mapping, which you can configure via the mappings APIs.
-The default percolate mapping doesn't index the query field, only stores it.
-
-Because `.percolate` is a type it also has a mapping. By default the following mapping is active:
-
-[source,js]
---------------------------------------------------
-{
-	".percolator" : {
-		"properties" : {
-			"query" : {
-				"type" : "object",
-				"enabled" : false
-			}
-		}
-	}
-}
---------------------------------------------------
-
-If needed, this mapping can be modified with the update mapping API.
-
-In order to un-register a percolate query the delete API can be used. So if the previous added query needs to be deleted
-the following delete requests needs to be executed:
-
-[source,js]
---------------------------------------------------
-curl -XDELETE localhost:9200/my-index/.percolator/1
---------------------------------------------------
-
-[float]
-=== Percolate API
-
-The percolate API executes in a distributed manner, meaning it executes on all shards an index points to.
-
-.Required options
-* `index` - The index that contains the `.percolator` type. This can also be an alias.
-* `type` - The type of the document to be percolated. The mapping of that type is used to parse document.
-* `doc` - The actual document to percolate. Unlike the other two options this needs to be specified in the request body. Note: This isn't required when percolating an existing document.
-
-[source,js]
---------------------------------------------------
-curl -XGET 'localhost:9200/twitter/tweet/_percolate' -d '{
-	"doc" : {
-		"created_at" : "2010-10-10T00:00:00",
-		"message" : "some text"
-	}
-}'
---------------------------------------------------
-
-.Additional supported query string options
-* `routing` - In case the percolate queries are partitioned by a custom routing value, that routing option makes sure
-that the percolate request only gets executed on the shard where the routing value is partitioned to. This means that
-the percolate request only gets executed on one shard instead of all shards. Multiple values can be specified as a
-comma separated string, in that case the request can be be executed on more than one shard.
-* `preference` - Controls which shard replicas are preferred to execute the request on. Works the same as in the search API.
-* `ignore_unavailable` - Controls if missing concrete indices should silently be ignored. Same as is in the search API.
-* `percolate_format` - If `ids` is specified then the matches array in the percolate response will contain a string
-array of the matching ids instead of an array of objects. This can be useful to reduce the amount of data being send
-back to the client. Obviously if there are two percolator queries with same id from different indices there is no way
-to find out which percolator query belongs to what index. Any other value to `percolate_format` will be ignored.
-
-.Additional request body options
-* `filter` - Reduces the number queries to execute during percolating. Only the percolator queries that match with the
-filter will be included in the percolate execution. The filter option works in near realtime, so a refresh needs to have
-occurred for the filter to included the latest percolate queries.
-* `query` - Same as the `filter` option, but also the score is computed. The computed scores can then be used by the
-`track_scores` and `sort` option.
-* `size` - Defines to maximum number of matches (percolate queries) to be returned. Defaults to 10.
-* `track_scores` - Whether the `_score` is included for each match. The `_score` is based on the query and represents
-how the query matched the *percolate query's metadata*, *not* how the document (that is being percolated) matched
-the query. The `query` option is required for this option. Defaults to `false`.
-* `sort` - Define a sort specification like in the search API. Currently only sorting `_score` reverse (default relevancy)
-is supported. Other sort fields will throw an exception. The `size` and `query` option are required for this setting. Like
-`track_score` the score is based on the query and represents how the query matched to the percolate query's metadata
-and *not* how the document being percolated matched to the query.
-* `aggs` - Allows aggregation definitions to be included. The aggregations are based on the matching percolator queries,
-look at the aggregation documentation on how to define aggregations.
-* `highlight` - Allows highlight definitions to be included. The document being percolated is being highlight for each
-matching query. This allows you to see how each match is highlighting the document being percolated. See highlight
-documentation on how to define highlights. The `size` option is required for highlighting, the performance of highlighting
- in the percolate API depends of how many matches are being highlighted.
-
-[float]
-=== Dedicated Percolator Index
-
-Percolate queries can be added to any index. Instead of adding percolate queries to the index the data resides in,
-these queries can also be added to a dedicated index. The advantage of this is that this dedicated percolator index
-can have its own index settings (For example the number of primary and replica shards). If you choose to have a dedicated
-percolate index, you need to make sure that the mappings from the normal index are also available on the percolate index.
-Otherwise percolate queries can be parsed incorrectly.
-
-[float]
-=== Filtering Executed Queries
-
-Filtering allows to reduce the number of queries, any filter that the search API supports, (except the ones mentioned in important notes)
-can also be used in the percolate API. The filter only works on the metadata fields. The `query` field isn't indexed by
-default. Based on the query we indexed before, the following filter can be defined:
-
-[source,js]
---------------------------------------------------
-curl -XGET localhost:9200/test/type1/_percolate -d '{
-    "doc" : {
-        "field" : "value"
-    },
-    "filter" : {
-        "term" : {
-            "priority" : "high"
-        }
-    }
-}'
---------------------------------------------------
-
-[float]
-=== Percolator Count API
-
-The count percolate API, only keeps track of the number of matches and doesn't keep track of the actual matches
-Example:
-
-[source,js]
---------------------------------------------------
-curl -XGET 'localhost:9200/my-index/my-type/_percolate/count' -d '{
-   "doc" : {
-       "message" : "some message"
-   }
-}'
---------------------------------------------------
-
-Response:
-
-[source,js]
---------------------------------------------------
-{
-   ... // header
-   "total" : 3
-}
---------------------------------------------------
-
-
-[float]
-=== Percolating an Existing Document
-
-In order to percolate a newly indexed document, the percolate existing document can be used. Based on the response
-from an index request, the `_id` and other meta information can be used to immediately percolate the newly added
-document.
-
-.Supported options for percolating an existing document on top of existing percolator options:
-* `id` - The id of the document to retrieve the source for.
-* `percolate_index` - The index containing the percolate queries. Defaults to the `index` defined in the url.
-* `percolate_type` - The percolate type (used for parsing the document). Default to `type` defined in the url.
-* `routing` - The routing value to use when retrieving the document to percolate.
-* `preference` - Which shard to prefer when retrieving the existing document.
-* `percolate_routing` - The routing value to use when percolating the existing document.
-* `percolate_preference` - Which shard to prefer when executing the percolate request.
-* `version` - Enables a version check. If the fetched document's version isn't equal to the specified version then the request fails with a version conflict and the percolation request is aborted.
-
-Internally the percolate API will issue a GET request for fetching the `_source` of the document to percolate.
-For this feature to work, the `_source` for documents to be percolated needs to be stored.
-
-If percolating an existing document and the a document is also specified in the source of the percolate request then
-an error is thrown. Either the document to percolate should be specified in the source or be defined by specifying the
-index, type and id.
-
-[float]
-==== Example
-
-Index response:
-
-[source,js]
---------------------------------------------------
-{
-	"_index" : "my-index",
-	"_type" : "message",
-	"_id" : "1",
-	"_version" : 1,
-	"created" : true
-}
---------------------------------------------------
-
-Percolating an Existing Document:
-
-[source,js]
---------------------------------------------------
-curl -XGET 'localhost:9200/my-index1/message/1/_percolate'
---------------------------------------------------
-
-The response is the same as with the regular percolate API.
-
-[float]
-=== Multi Percolate API
-
-The multi percolate API allows to bundle multiple percolate requests into a single request, similar to what the multi
-search API does to search requests. The request body format is line based. Each percolate request item takes two lines,
-the first line is the header and the second line is the body.
-
-The header can contain any parameter that normally would be set via the request path or query string parameters.
-There are several percolate actions, because there are multiple types of percolate requests.
-
-.Supported actions:
-* `percolate` - Action for defining a regular percolate request.
-* `count` - Action for defining a count percolate request.
-
-Depending on the percolate action different parameters can be specified. For example the percolate and percolate existing
-document actions support different parameters.
-
-.The following endpoints are supported
-* `GET|POST /[index]/[type]/_mpercolate`
-* `GET|POST /[index]/_mpercolate`
-* `GET|POST /_mpercolate`
-
-The `index` and `type` defined in the url path are the default index and type.
-
-[float]
-==== Example
-
-Request:
-
-[source,js]
---------------------------------------------------
-curl -XGET 'localhost:9200/twitter/tweet/_mpercolate' --data-binary "@requests.txt"; echo
---------------------------------------------------
-
-The index `twitter` is the default index, and the type `tweet` is the default type and will be used in the case a header
-doesn't specify an index or type.
-
-requests.txt:
-
-[source,js]
---------------------------------------------------
-{"percolate" : {"index" : "twitter", "type" : "tweet"}}
-{"doc" : {"message" : "some text"}}
-{"percolate" : {"index" : "twitter", "type" : "tweet", "id" : "1"}}
-{"percolate" : {"index" : "users", "type" : "user", "id" : "3", "percolate_index" : "users_2012" }}
-{"size" : 10}
-{"count" : {"index" : "twitter", "type" : "tweet"}}
-{"doc" : {"message" : "some other text"}}
-{"count" : {"index" : "twitter", "type" : "tweet", "id" : "1"}}
---------------------------------------------------
-
-For a percolate existing document item (headers with the `id` field), the response can be an empty JSON object.
-All the required options are set in the header.
-
-Response:
-
-[source,js]
---------------------------------------------------
-{
-    "responses" : [
-        {
-            "took" : 24,
-            "_shards" : {
-                "total" : 5,
-                "successful" : 5,
-                "failed" : 0,
-            },
-            "total" : 3,
-            "matches" : [
-                {
-                    "_index": "twitter",
-                    "_id": "1"
-                },
-                {
-                    "_index": "twitter",
-                    "_id": "2"
-                },
-                {
-                    "_index": "twitter",
-                    "_id": "3"
-                }
-            ]
-        },
-        {
-            "took" : 12,
-            "_shards" : {
-                "total" : 5,
-                "successful" : 5,
-                "failed" : 0,
-            },
-            "total" : 3,
-            "matches" : [
-                {
-                    "_index": "twitter",
-                    "_id": "4"
-                },
-                {
-                    "_index": "twitter",
-                    "_id": "5"
-                },
-                {
-                    "_index": "twitter",
-                    "_id": "6"
-                }
-             ]
-        },
-        {
-            "error" : "DocumentMissingException[[_na][_na] [user][3]: document missing]"
-        },
-        {
-            "took" : 12,
-            "_shards" : {
-                "total" : 5,
-                "successful" : 5,
-                "failed" : 0,
-            },
-            "total" : 3
-        },
-        {
-            "took" : 14,
-            "_shards" : {
-                "total" : 5,
-                "successful" : 5,
-                "failed" : 0,
-            },
-            "total" : 3
-        }
-    ]
-}
-
---------------------------------------------------
-
-Each item represents a percolate response, the order of the items maps to the order in which the percolate requests
-were specified. In case a percolate request failed, the item response is substituted with an error message.
-
-[float]
-=== How it Works Under the Hood
-
-When indexing a document that contains a query in an index and the `.percolator` type, the query part of the documents gets
-parsed into a Lucene query and is kept in memory until that percolator document is removed or the index containing the
-`.percolator` type gets removed. So, all the active percolator queries are kept in memory.
-
-At percolate time, the document specified in the request gets parsed into a Lucene document and is stored in a in-memory
-Lucene index. This in-memory index can just hold this one document and it is optimized for that. Then all the queries
-that are registered to the index that the percolate request is targeted for, are going to be executed on this single document
-in-memory index. This happens on each shard the percolate request needs to execute.
-
-By using `routing`, `filter` or `query` features the amount of queries that need to be executed can be reduced and thus
-the time the percolate API needs to run can be decreased.
-
-[float]
-=== Important Notes
-
-Because the percolator API is processing one document at a time, it doesn't support queries and filters that run
-against child documents such as `has_child` and `has_parent`.
-
-The `inner_hits` feature on the `nested` query isn't supported in the percolate api.
-
-The `wildcard` and `regexp` query natively use a lot of memory and because the percolator keeps the queries into memory
-this can easily take up the available memory in the heap space. If possible try to use a `prefix` query or ngramming to
-achieve the same result (with way less memory being used).
-
-The `delete-by-query` plugin doesn't work to unregister a query, it only deletes the percolate documents from disk. In order
-to update the registered queries in memory the index needs be closed and opened.
-
-[float]
-=== Forcing Unmapped Fields to be Handled as Strings
-
-In certain cases it is unknown what kind of percolator queries do get registered, and if no field mapping exists for fields
-that are referred by percolator queries then adding a percolator query fails. This means the mapping needs to be updated
-to have the field with the appropriate settings, and then the percolator query can be added. But sometimes it is sufficient
-if all unmapped fields are handled as if these were default string fields. In those cases one can configure the
-`index.percolator.map_unmapped_fields_as_string` setting to `true` (default to `false`) and then if a field referred in
-a percolator query does not exist, it will be handled as a default string field so that adding the percolator query doesn't
-fail.
+added[5.0.0,For indices created on or after version 5.0.0-alpha1 the percolator automatically indexes the query terms with the percolator queries. This allows the percolator to percolate documents more quickly. It is advisable to reindex any pre 5.0.0 indices to take advantage of this new optimization]
diff --git a/docs/reference/setup.asciidoc b/docs/reference/setup.asciidoc
index 15f23e6fe1e..f0bf03985b0 100644
--- a/docs/reference/setup.asciidoc
+++ b/docs/reference/setup.asciidoc
@@ -67,13 +67,12 @@ There are added features when using the `elasticsearch` shell script.
 The first, which was explained earlier, is the ability to easily run the
 process either in the foreground or the background.
 
-Another feature is the ability to pass `-D` or getopt long style
-configuration parameters directly to the script. When set, all override
-anything set using either `JAVA_OPTS` or `ES_JAVA_OPTS`. For example:
+Another feature is the ability to pass `-E` configuration parameters
+directly to the script. For example:
 
 [source,sh]
 --------------------------------------------------
-$ bin/elasticsearch -Des.index.refresh_interval=5s --node.name=my-node
+$ bin/elasticsearch -Ees.index.refresh_interval=5s -Ees.node.name=my-node
 --------------------------------------------------
 *************************************************************************
 
diff --git a/docs/reference/setup/configuration.asciidoc b/docs/reference/setup/configuration.asciidoc
index bef563cd965..1a687f15fb9 100644
--- a/docs/reference/setup/configuration.asciidoc
+++ b/docs/reference/setup/configuration.asciidoc
@@ -259,7 +259,7 @@ command, for example:
 
 [source,sh]
 --------------------------------------------------
-$ elasticsearch -Des.network.host=10.0.0.4
+$ elasticsearch -Ees.network.host=10.0.0.4
 --------------------------------------------------
 
 Another option is to set `es.default.` prefix instead of `es.` prefix,
@@ -336,7 +336,7 @@ course, the above can also be set as a "collapsed" setting, for example:
 
 [source,sh]
 --------------------------------------------------
-$ elasticsearch -Des.index.refresh_interval=5s
+$ elasticsearch -Ees.index.refresh_interval=5s
 --------------------------------------------------
 
 All of the index level configuration can be found within each
diff --git a/docs/reference/setup/rolling_upgrade.asciidoc b/docs/reference/setup/rolling_upgrade.asciidoc
index b3c00d337f8..cb9073b558e 100644
--- a/docs/reference/setup/rolling_upgrade.asciidoc
+++ b/docs/reference/setup/rolling_upgrade.asciidoc
@@ -80,7 +80,7 @@ To upgrade using a zip or compressed tarball:
     overwrite the `config` or `data` directories.
 
 *   Either copy the files in the `config` directory from your old installation
-    to your new installation, or use the `--path.conf` option on the command
+    to your new installation, or use the `-E path.conf=` option on the command
     line to point to an external config directory.
 
 *   Either copy the files in the `data` directory from your old installation
diff --git a/modules/build.gradle b/modules/build.gradle
index 3cafe7d903f..558163a420a 100644
--- a/modules/build.gradle
+++ b/modules/build.gradle
@@ -35,12 +35,6 @@ subprojects {
     throw new InvalidModelException("Modules cannot contain config files") 
   }
 
-  project.afterEvaluate {
-    if (esplugin.isolated == false) {
-      throw new InvalidModelException("Modules cannot disable isolation")
-    }
-  }
-
   // these are implementation details of our build, no need to publish them!
   install.enabled = false
   uploadArchives.enabled = false
diff --git a/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/10_basic.yaml b/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/10_basic.yaml
index 5c0cca3772e..ebb310ecf7a 100644
--- a/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/10_basic.yaml
+++ b/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/10_basic.yaml
@@ -9,3 +9,18 @@
         nodes.info: {}
 
     - match:  { nodes.$master.modules.0.name: ingest-grok  }
+    - match:  { nodes.$master.ingest.processors.0.type: append }
+    - match:  { nodes.$master.ingest.processors.1.type: convert }
+    - match:  { nodes.$master.ingest.processors.2.type: date }
+    - match:  { nodes.$master.ingest.processors.3.type: fail }
+    - match:  { nodes.$master.ingest.processors.4.type: foreach }
+    - match:  { nodes.$master.ingest.processors.5.type: grok }
+    - match:  { nodes.$master.ingest.processors.6.type: gsub }
+    - match:  { nodes.$master.ingest.processors.7.type: join }
+    - match:  { nodes.$master.ingest.processors.8.type: lowercase }
+    - match:  { nodes.$master.ingest.processors.9.type: remove }
+    - match:  { nodes.$master.ingest.processors.10.type: rename }
+    - match:  { nodes.$master.ingest.processors.11.type: set }
+    - match:  { nodes.$master.ingest.processors.12.type: split }
+    - match:  { nodes.$master.ingest.processors.13.type: trim }
+    - match:  { nodes.$master.ingest.processors.14.type: uppercase }
diff --git a/modules/lang-expression/licenses/lucene-expressions-6.0.0-snapshot-bea235f.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index d9a29f17c50..00000000000
--- a/modules/lang-expression/licenses/lucene-expressions-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8d11bf581b0afc25f87a57c06834cd85930d2ffa
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-6.0.0-snapshot-f0aa4fc.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..5237907f224
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+f36f8010c9fec7342d34bece819c13de5f241135
\ No newline at end of file
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
index 0f56adeea55..3944090cef2 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
@@ -112,14 +112,16 @@ class ExpressionSearchScript implements SearchScript {
 
             @Override
             public void setNextVar(String name, Object value) {
-                assert(specialValue != null);
                 // this should only be used for the special "_value" variable used in aggregations
                 assert(name.equals("_value"));
 
-                if (value instanceof Number) {
-                    specialValue.setValue(((Number)value).doubleValue());
-                } else {
-                    throw new ScriptException("Cannot use expression with text variable using " + compiledScript);
+                // _value isn't used in script if specialValue == null
+                if (specialValue != null) {
+                    if (value instanceof Number) {
+                        specialValue.setValue(((Number)value).doubleValue());
+                    } else {
+                        throw new ScriptException("Cannot use expression with text variable using " + compiledScript);
+                    }
                 }
             }
         };
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
index 5246d0dc306..a8856ea78b5 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
@@ -383,7 +383,11 @@ public class MoreExpressionTests extends ESIntegTestCase {
                                 .script(new Script("_value * 3", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null)))
                 .addAggregation(
                         AggregationBuilders.stats("double_agg").field("y")
-                                .script(new Script("_value - 1.1", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null)));
+                                .script(new Script("_value - 1.1", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null)))
+                .addAggregation(
+                        AggregationBuilders.stats("const_agg").field("x") // specifically to test a script w/o _value
+                                .script(new Script("3.0", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null))
+                );
 
         SearchResponse rsp = req.get();
         assertEquals(3, rsp.getHits().getTotalHits());
@@ -395,6 +399,11 @@ public class MoreExpressionTests extends ESIntegTestCase {
         stats = rsp.getAggregations().get("double_agg");
         assertEquals(0.7, stats.getMax(), 0.0001);
         assertEquals(0.1, stats.getMin(), 0.0001);
+
+        stats = rsp.getAggregations().get("const_agg");
+        assertThat(stats.getMax(), equalTo(3.0));
+        assertThat(stats.getMin(), equalTo(3.0));
+        assertThat(stats.getAvg(), equalTo(3.0));
     }
 
     public void testStringSpecialValueVariable() throws Exception {
diff --git a/modules/lang-groovy/build.gradle b/modules/lang-groovy/build.gradle
index 89444a4e926..884fe8b65ba 100644
--- a/modules/lang-groovy/build.gradle
+++ b/modules/lang-groovy/build.gradle
@@ -28,8 +28,8 @@ dependencies {
 
 integTest {
   cluster {
-    systemProperty 'es.script.inline', 'true'
-    systemProperty 'es.script.indexed', 'true'
+    setting 'script.inline', 'true'
+    setting 'script.indexed', 'true'
   }
 }
 
@@ -38,6 +38,17 @@ thirdPartyAudit.excludes = [
   // for example we do not need ivy, scripts arent allowed to download code
   'com.thoughtworks.xstream.XStream', 
   'groovyjarjarasm.asm.util.Textifiable', 
+  // commons-cli is referenced by groovy, even though they supposedly
+  // jarjar it. Since we don't use the cli, we don't need the dep.
+  'org.apache.commons.cli.CommandLine',
+  'org.apache.commons.cli.CommandLineParser',
+  'org.apache.commons.cli.GnuParser',
+  'org.apache.commons.cli.HelpFormatter',
+  'org.apache.commons.cli.Option',
+  'org.apache.commons.cli.OptionBuilder',
+  'org.apache.commons.cli.Options',
+  'org.apache.commons.cli.Parser',
+  'org.apache.commons.cli.PosixParser',
   'org.apache.ivy.Ivy', 
   'org.apache.ivy.core.event.IvyListener', 
   'org.apache.ivy.core.event.download.PrepareDownloadEvent', 
diff --git a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
index 4d9e7a4b57b..60a8a0c1338 100644
--- a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
+++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
@@ -316,7 +316,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
                 });
             } catch (Throwable e) {
                 if (logger.isTraceEnabled()) {
-                    logger.trace("failed to run " + compiledScript, e);
+                    logger.trace("failed to run {}", e, compiledScript);
                 }
                 throw new ScriptException("failed to run " + compiledScript, e);
             }
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java
index 2fb0f9f6327..6f83746d4ce 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java
@@ -332,7 +332,7 @@ public class EquivalenceTests extends ESIntegTestCase {
         createIndex("idx");
 
         final int numDocs = scaledRandomIntBetween(2500, 5000);
-        logger.info("Indexing [" + numDocs +"] docs");
+        logger.info("Indexing [{}] docs", numDocs);
         List<IndexRequestBuilder> indexingRequests = new ArrayList<>();
         for (int i = 0; i < numDocs; ++i) {
             indexingRequests.add(client().prepareIndex("idx", "type", Integer.toString(i)).setSource("double_value", randomDouble()));
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java
index e717ea6d6fb..4642d4662c9 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java
@@ -543,7 +543,7 @@ public class ExtendedStatsTests extends AbstractNumericTestCase {
         ShardSearchFailure[] failures = response.getShardFailures();
         if (failures.length != expectedFailures) {
             for (ShardSearchFailure failure : failures) {
-                logger.error("Shard Failure: {}", failure.reason(), failure.toString());
+                logger.error("Shard Failure: {}", failure.getCause(), failure);
             }
             fail("Unexpected shard failures!");
         }
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoShapeIntegrationTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoShapeIntegrationTests.java
index 98a23b3e1fd..8a86a0a1fb4 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoShapeIntegrationTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoShapeIntegrationTests.java
@@ -76,7 +76,7 @@ public class GeoShapeIntegrationTests extends ESIntegTestCase {
 
         // left orientation test
         IndicesService indicesService = internalCluster().getInstance(IndicesService.class, findNodeName(idxName));
-        IndexService indexService = indicesService.indexService(idxName);
+        IndexService indexService = indicesService.indexService(resolveIndex(idxName));
         MappedFieldType fieldType = indexService.mapperService().fullName("location");
         assertThat(fieldType, instanceOf(GeoShapeFieldMapper.GeoShapeFieldType.class));
 
@@ -88,7 +88,7 @@ public class GeoShapeIntegrationTests extends ESIntegTestCase {
 
         // right orientation test
         indicesService = internalCluster().getInstance(IndicesService.class, findNodeName(idxName+"2"));
-        indexService = indicesService.indexService(idxName+"2");
+        indexService = indicesService.indexService(resolveIndex((idxName+"2")));
         fieldType = indexService.mapperService().fullName("location");
         assertThat(fieldType, instanceOf(GeoShapeFieldMapper.GeoShapeFieldType.class));
 
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
index 60b9460bb12..c8b30a5e2de 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
@@ -67,15 +67,9 @@ import org.elasticsearch.action.get.MultiGetAction;
 import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.index.IndexAction;
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.percolate.MultiPercolateAction;
-import org.elasticsearch.action.percolate.MultiPercolateRequest;
-import org.elasticsearch.action.percolate.PercolateAction;
-import org.elasticsearch.action.percolate.PercolateRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.action.suggest.SuggestAction;
-import org.elasticsearch.action.suggest.SuggestRequest;
 import org.elasticsearch.action.termvectors.MultiTermVectorsAction;
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
 import org.elasticsearch.action.termvectors.TermVectorsAction;
@@ -85,7 +79,6 @@ import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -146,6 +139,9 @@ public class IndicesRequestTests extends ESIntegTestCase {
     protected Settings nodeSettings(int ordinal) {
         // must set this independently of the plugin so it overrides MockTransportService
         return Settings.builder().put(super.nodeSettings(ordinal))
+            // InternalClusterInfoService sends IndicesStatsRequest periodically which messes with this test
+            // this setting disables it...
+            .put("cluster.routing.allocation.disk.threshold_enabled", false)
             .put(NetworkModule.TRANSPORT_SERVICE_TYPE_KEY, "intercepting").build();
     }
 
@@ -370,7 +366,7 @@ public class IndicesRequestTests extends ESIntegTestCase {
         internalCluster().clientNodeClient().admin().indices().flush(flushRequest).actionGet();
 
         clearInterceptedActions();
-        String[] indices = new IndexNameExpressionResolver(Settings.EMPTY).concreteIndices(client().admin().cluster().prepareState().get().getState(), flushRequest);
+        String[] indices = new IndexNameExpressionResolver(Settings.EMPTY).concreteIndexNames(client().admin().cluster().prepareState().get().getState(), flushRequest);
         assertIndicesSubset(Arrays.asList(indices), indexShardActions);
     }
 
@@ -393,7 +389,7 @@ public class IndicesRequestTests extends ESIntegTestCase {
         internalCluster().clientNodeClient().admin().indices().refresh(refreshRequest).actionGet();
 
         clearInterceptedActions();
-        String[] indices = new IndexNameExpressionResolver(Settings.EMPTY).concreteIndices(client().admin().cluster().prepareState().get().getState(), refreshRequest);
+        String[] indices = new IndexNameExpressionResolver(Settings.EMPTY).concreteIndexNames(client().admin().cluster().prepareState().get().getState(), refreshRequest);
         assertIndicesSubset(Arrays.asList(indices), indexShardActions);
     }
 
@@ -441,17 +437,6 @@ public class IndicesRequestTests extends ESIntegTestCase {
         assertSameIndices(indicesStatsRequest, indicesStats);
     }
 
-    public void testSuggest() {
-        String suggestAction = SuggestAction.NAME + "[s]";
-        interceptTransportActions(suggestAction);
-
-        SuggestRequest suggestRequest = new SuggestRequest(randomIndicesOrAliases());
-        internalCluster().clientNodeClient().suggest(suggestRequest).actionGet();
-
-        clearInterceptedActions();
-        assertSameIndices(suggestRequest, suggestAction);
-    }
-
     public void testValidateQuery() {
         String validateQueryShardAction = ValidateQueryAction.NAME + "[s]";
         interceptTransportActions(validateQueryShardAction);
@@ -463,51 +448,6 @@ public class IndicesRequestTests extends ESIntegTestCase {
         assertSameIndices(validateQueryRequest, validateQueryShardAction);
     }
 
-    public void testPercolate() {
-        String percolateShardAction = PercolateAction.NAME + "[s]";
-        interceptTransportActions(percolateShardAction);
-
-        client().prepareIndex("test-get", "type", "1").setSource("field","value").get();
-
-        PercolateRequest percolateRequest = new PercolateRequest().indices(randomIndicesOrAliases()).documentType("type");
-        if (randomBoolean()) {
-            percolateRequest.getRequest(new GetRequest("test-get", "type", "1"));
-        } else {
-            percolateRequest.source("\"field\":\"value\"");
-        }
-        internalCluster().clientNodeClient().percolate(percolateRequest).actionGet();
-
-        clearInterceptedActions();
-        assertSameIndices(percolateRequest, percolateShardAction);
-    }
-
-    public void testMultiPercolate() {
-        String multiPercolateShardAction = MultiPercolateAction.NAME + "[shard][s]";
-        interceptTransportActions(multiPercolateShardAction);
-
-        client().prepareIndex("test-get", "type", "1").setSource("field", "value").get();
-
-        MultiPercolateRequest multiPercolateRequest = new MultiPercolateRequest();
-        List<String> indices = new ArrayList<>();
-        int numRequests = iterations(1, 30);
-        for (int i = 0; i < numRequests; i++) {
-            String[] indicesOrAliases = randomIndicesOrAliases();
-            Collections.addAll(indices, indicesOrAliases);
-            PercolateRequest percolateRequest = new PercolateRequest().indices(indicesOrAliases).documentType("type");
-            if (randomBoolean()) {
-                percolateRequest.getRequest(new GetRequest("test-get", "type", "1"));
-            } else {
-                percolateRequest.source("\"field\":\"value\"");
-            }
-            multiPercolateRequest.add(percolateRequest);
-        }
-
-        internalCluster().clientNodeClient().multiPercolate(multiPercolateRequest).actionGet();
-
-        clearInterceptedActions();
-        assertIndicesSubset(indices, multiPercolateShardAction);
-    }
-
     public void testOpenIndex() {
         interceptTransportActions(OpenIndexAction.NAME);
 
@@ -785,8 +725,8 @@ public class IndicesRequestTests extends ESIntegTestCase {
         private final Map<String, List<TransportRequest>> requests = new HashMap<>();
 
         @Inject
-        public InterceptingTransportService(Settings settings, Transport transport, ThreadPool threadPool, NamedWriteableRegistry namedWriteableRegistry) {
-            super(settings, transport, threadPool, namedWriteableRegistry);
+        public InterceptingTransportService(Settings settings, Transport transport, ThreadPool threadPool) {
+            super(settings, transport, threadPool);
         }
 
         synchronized List<TransportRequest> consumeRequests(String action) {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java
index 67f7d6ff0da..592fb362bc2 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/RandomScoreFunctionTests.java
@@ -316,10 +316,9 @@ public class RandomScoreFunctionTests extends ESIntegTestCase {
             }
         }
 
-        System.out.println();
-        System.out.println("max repeat: " + maxRepeat);
-        System.out.println("avg repeat: " + sumRepeat / (double) filled);
-        System.out.println("distribution: " + filled / (double) count);
+        logger.info("max repeat: {}", maxRepeat);
+        logger.info("avg repeat: {}", sumRepeat / (double) filled);
+        logger.info("distribution: {}", filled / (double) count);
 
         int percentile50 = filled / 2;
         int percentile25 = (filled / 4);
@@ -333,18 +332,18 @@ public class RandomScoreFunctionTests extends ESIntegTestCase {
             }
             sum += i * matrix[i];
             if (percentile50 == 0) {
-                System.out.println("median: " + i);
+                logger.info("median: {}", i);
             } else if (percentile25 == 0) {
-                System.out.println("percentile_25: " + i);
+                logger.info("percentile_25: {}", i);
             } else if (percentile75 == 0) {
-                System.out.println("percentile_75: " + i);
+                logger.info("percentile_75: {}", i);
             }
             percentile50--;
             percentile25--;
             percentile75--;
         }
 
-        System.out.println("mean: " + sum / (double) count);
+        logger.info("mean: {}", sum / (double) count);
     }
 
 }
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java
index 752165902ed..aa47fe98bb3 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptQuerySearchTests.java
@@ -50,8 +50,8 @@ public class ScriptQuerySearchTests extends ESIntegTestCase {
     }
 
     @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.settingsBuilder().put(super.nodeSettings(nodeOrdinal))
+    public Settings indexSettings() {
+        return Settings.settingsBuilder().put(super.indexSettings())
                 // aggressive filter caching so that we can assert on the number of iterations of the script filters
                 .put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), IndexModule.INDEX_QUERY_CACHE)
                 .put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)
@@ -80,9 +80,9 @@ public class ScriptQuerySearchTests extends ESIntegTestCase {
 
         assertThat(response.getHits().totalHits(), equalTo(2L));
         assertThat(response.getHits().getAt(0).id(), equalTo("2"));
-        assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0));
+        assertThat(response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0));
         assertThat(response.getHits().getAt(1).id(), equalTo("3"));
-        assertThat((Double) response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(3.0));
+        assertThat(response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(3.0));
 
         Map<String, Object> params = new HashMap<>();
         params.put("param1", 2);
@@ -95,7 +95,7 @@ public class ScriptQuerySearchTests extends ESIntegTestCase {
 
         assertThat(response.getHits().totalHits(), equalTo(1L));
         assertThat(response.getHits().getAt(0).id(), equalTo("3"));
-        assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(3.0));
+        assertThat(response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(3.0));
 
         params = new HashMap<>();
         params.put("param1", -1);
@@ -108,11 +108,11 @@ public class ScriptQuerySearchTests extends ESIntegTestCase {
 
         assertThat(response.getHits().totalHits(), equalTo(3L));
         assertThat(response.getHits().getAt(0).id(), equalTo("1"));
-        assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0));
+        assertThat(response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0));
         assertThat(response.getHits().getAt(1).id(), equalTo("2"));
-        assertThat((Double) response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0));
+        assertThat(response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0));
         assertThat(response.getHits().getAt(2).id(), equalTo("3"));
-        assertThat((Double) response.getHits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0));
+        assertThat(response.getHits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0));
     }
 
     private static AtomicInteger scriptCounter = new AtomicInteger(0);
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java
index 72abe487d89..7838bb58f8e 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java
@@ -116,7 +116,7 @@ public class SearchStatsTests extends ESIntegTestCase {
         }
 
         IndicesStatsResponse indicesStats = client().admin().indices().prepareStats().execute().actionGet();
-        logger.debug("###### indices search stats: " + indicesStats.getTotal().getSearch());
+        logger.debug("###### indices search stats: {}", indicesStats.getTotal().getSearch());
         assertThat(indicesStats.getTotal().getSearch().getTotal().getQueryCount(), greaterThan(0L));
         assertThat(indicesStats.getTotal().getSearch().getTotal().getQueryTimeInMillis(), greaterThan(0L));
         assertThat(indicesStats.getTotal().getSearch().getTotal().getFetchCount(), greaterThan(0L));
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java
index 8cb8ebcc0a4..3aa3f1422ec 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java
@@ -28,6 +28,7 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.groovy.GroovyPlugin;
 import org.elasticsearch.search.sort.ScriptSortBuilder;
+import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -65,14 +66,14 @@ public class SimpleSortTests extends ESIntegTestCase {
         Random random = random();
         assertAcked(prepareCreate("test")
                 .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
-                        .startObject("str_value").field("type", "keyword").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject()
+                        .startObject("str_value").field("type", "keyword").endObject()
                         .startObject("boolean_value").field("type", "boolean").endObject()
-                        .startObject("byte_value").field("type", "byte").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject()
-                        .startObject("short_value").field("type", "short").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject()
-                        .startObject("integer_value").field("type", "integer").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject()
-                        .startObject("long_value").field("type", "long").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject()
-                        .startObject("float_value").field("type", "float").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject()
-                        .startObject("double_value").field("type", "double").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject()
+                        .startObject("byte_value").field("type", "byte").endObject()
+                        .startObject("short_value").field("type", "short").endObject()
+                        .startObject("integer_value").field("type", "integer").endObject()
+                        .startObject("long_value").field("type", "long").endObject()
+                        .startObject("float_value").field("type", "float").endObject()
+                        .startObject("double_value").field("type", "double").endObject()
                         .endObject().endObject().endObject()));
         ensureGreen();
         List<IndexRequestBuilder> builders = new ArrayList<>();
@@ -109,7 +110,7 @@ public class SimpleSortTests extends ESIntegTestCase {
         SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
                 .setSize(size)
-                .addSort(new ScriptSortBuilder(new Script("doc['str_value'].value"), "string")).execute().actionGet();
+                .addSort(new ScriptSortBuilder(new Script("doc['str_value'].value"), ScriptSortType.STRING)).execute().actionGet();
         assertHitCount(searchResponse, 10);
         assertThat(searchResponse.getHits().hits().length, equalTo(size));
         for (int i = 0; i < size; i++) {
@@ -217,7 +218,7 @@ public class SimpleSortTests extends ESIntegTestCase {
 
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(20L));
         for (int i = 0; i < 10; i++) {
-            assertThat("res: " + i + " id: " + searchResponse.getHits().getAt(i).getId(), (Double) searchResponse.getHits().getAt(i).field("min").value(), closeTo((double) i, TOLERANCE));
+            assertThat("res: " + i + " id: " + searchResponse.getHits().getAt(i).getId(), (Double) searchResponse.getHits().getAt(i).field("min").value(), closeTo(i, TOLERANCE));
         }
     }
 
@@ -226,7 +227,7 @@ public class SimpleSortTests extends ESIntegTestCase {
         // We have to specify mapping explicitly because by the time search is performed dynamic mapping might not
         // be propagated to all nodes yet and sort operation fail when the sort field is not defined
         String mapping = jsonBuilder().startObject().startObject("type1").startObject("properties")
-                .startObject("svalue").field("type", "keyword").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject()
+                .startObject("svalue").field("type", "keyword").endObject()
                 .endObject().endObject().endObject().string();
         assertAcked(prepareCreate("test").addMapping("type1", mapping));
         ensureGreen();
@@ -326,7 +327,7 @@ public class SimpleSortTests extends ESIntegTestCase {
         }
         refresh();
         SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery())
-                .addSort(SortBuilders.scriptSort(new Script("\u0027\u0027"), "string")).setSize(10).execute().actionGet();
+                .addSort(SortBuilders.scriptSort(new Script("\u0027\u0027"), ScriptSortType.STRING)).setSize(10).execute().actionGet();
         assertNoFailures(searchResponse);
     }
 }
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java
index 4934d2ae6c4..b06d3395b2b 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java
@@ -404,10 +404,10 @@ public class StatsTests extends AbstractNumericTestCase {
         ShardSearchFailure[] failures = response.getShardFailures();
         if (failures.length != expectedFailures) {
             for (ShardSearchFailure failure : failures) {
-                logger.error("Shard Failure: {}", failure.reason(), failure.toString());
+                logger.error("Shard Failure: {}", failure.getCause(), failure);
             }
             fail("Unexpected shard failures!");
         }
         assertThat("Not all shards are initialized", response.getSuccessfulShards(), equalTo(response.getTotalShards()));
     }
-}
\ No newline at end of file
+}
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java
index edf8be49ddc..606b26f193d 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.internal.IndexFieldMapper;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
@@ -243,7 +242,7 @@ public class StringTermsTests extends AbstractTermsTestCase {
         ExecutionMode[] executionModes = new ExecutionMode[] { null, ExecutionMode.GLOBAL_ORDINALS, ExecutionMode.GLOBAL_ORDINALS_HASH,
                 ExecutionMode.GLOBAL_ORDINALS_LOW_CARDINALITY };
         for (ExecutionMode executionMode : executionModes) {
-            logger.info("Execution mode:" + executionMode);
+            logger.info("Execution mode: {}", executionMode);
             SearchResponse response = client()
                     .prepareSearch("idx")
                     .setTypes("type")
@@ -1447,13 +1446,6 @@ public class StringTermsTests extends AbstractTermsTestCase {
             assertThat(bucket.getDocCount(), equalTo(i == 0 ? 5L : 2L));
             i++;
         }
-
-        response = client().prepareSearch("idx", "empty_bucket_idx").setTypes("type")
-                .addAggregation(terms("terms").executionHint(randomExecutionHint()).field(FieldNamesFieldMapper.NAME)).execute()
-                .actionGet();
-        assertSearchResponse(response);
-        terms = response.getAggregations().get("terms");
-        assertEquals(5L, terms.getBucketByKey("i").getDocCount());
     }
 
     public void testOtherDocCount() {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java
index dcc3abf2e1c..4e13219bae5 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java
@@ -28,6 +28,7 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -68,7 +69,8 @@ public class GroovyScriptTests extends ESIntegTestCase {
     public void assertScript(String scriptString) {
         Script script = new Script(scriptString, ScriptType.INLINE, "groovy", null);
         SearchResponse resp = client().prepareSearch("test")
-                .setSource(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).sort(SortBuilders.scriptSort(script, "number")))
+                .setSource(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).sort(SortBuilders.
+                        scriptSort(script, ScriptSortType.NUMBER)))
                 .get();
         assertNoFailures(resp);
     }
diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle
index f41ffb90128..8eed31dd668 100644
--- a/modules/lang-mustache/build.gradle
+++ b/modules/lang-mustache/build.gradle
@@ -28,7 +28,7 @@ dependencies {
 
 integTest {
   cluster {
-    systemProperty 'es.script.inline', 'true'
-    systemProperty 'es.script.indexed', 'true'
+    setting 'script.inline', 'true'
+    setting 'script.indexed', 'true'
   }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
index 766c5bff9c4..647a727b2dd 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
@@ -191,7 +191,7 @@ public final class MustacheScriptEngineService extends AbstractComponent impleme
                     }
                 });
             } catch (Exception e) {
-                logger.error("Error running " + template, e);
+                logger.error("Error running {}", e, template);
                 throw new ScriptException("Error running " + template, e);
             }
             return result.bytes();
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java
index f17bb7ea59f..79cac3181ea 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java
@@ -26,7 +26,6 @@ import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.search.suggest.SuggestBuilders.phraseSuggestion;
 import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion;
-import static org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.candidateGenerator;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestion;
@@ -50,6 +49,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.concurrent.ExecutionException;
 
 import org.elasticsearch.ElasticsearchException;
@@ -60,19 +60,22 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.action.suggest.SuggestRequestBuilder;
-import org.elasticsearch.action.suggest.SuggestResponse;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.mustache.MustachePlugin;
+import org.elasticsearch.search.suggest.SortBy;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.SuggestBuilder;
-import org.elasticsearch.search.suggest.SuggestBuilder.SuggestionBuilder;
+import org.elasticsearch.search.suggest.SuggestionBuilder;
 import org.elasticsearch.search.suggest.phrase.DirectCandidateGeneratorBuilder;
+import org.elasticsearch.search.suggest.phrase.Laplace;
+import org.elasticsearch.search.suggest.phrase.LinearInterpolation;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
+import org.elasticsearch.search.suggest.phrase.StupidBackoff;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
+import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 
@@ -99,12 +102,11 @@ public class SuggestSearchTests extends ESIntegTestCase {
         index("test", "type1", "4", "text", "abcc");
         refresh();
 
-        TermSuggestionBuilder termSuggest = termSuggestion("test")
-                .suggestMode("always") // Always, otherwise the results can vary between requests.
-                .text("abcd")
-                .field("text");
+        TermSuggestionBuilder termSuggest = termSuggestion("text")
+                .suggestMode(TermSuggestionBuilder.SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
+                .text("abcd");
         logger.info("--> run suggestions with one index");
-        searchSuggest( termSuggest);
+        searchSuggest("test", termSuggest);
         createIndex("test_1");
         ensureGreen();
 
@@ -113,13 +115,12 @@ public class SuggestSearchTests extends ESIntegTestCase {
         index("test_1", "type1", "3", "text", "ab bd");
         index("test_1", "type1", "4", "text", "ab cc");
         refresh();
-        termSuggest = termSuggestion("test")
-                .suggestMode("always") // Always, otherwise the results can vary between requests.
+        termSuggest = termSuggestion("text")
+                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
                 .text("ab cd")
-                .minWordLength(1)
-                .field("text");
+                .minWordLength(1);
         logger.info("--> run suggestions with two indices");
-        searchSuggest( termSuggest);
+        searchSuggest("test", termSuggest);
 
 
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
@@ -140,14 +141,13 @@ public class SuggestSearchTests extends ESIntegTestCase {
         index("test_2", "type1", "4", "text", "abcc");
         refresh();
 
-        termSuggest = termSuggestion("test")
-                .suggestMode("always") // Always, otherwise the results can vary between requests.
+        termSuggest = termSuggestion("text")
+                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
                 .text("ab cd")
-                .minWordLength(1)
-                .field("text");
+                .minWordLength(1);
         logger.info("--> run suggestions with three indices");
         try {
-            searchSuggest( termSuggest);
+            searchSuggest("test", termSuggest);
             fail(" can not suggest across multiple indices with different analysis chains");
         } catch (ReduceSearchPhaseException ex) {
             assertThat(ex.getCause(), instanceOf(IllegalStateException.class));
@@ -160,14 +160,13 @@ public class SuggestSearchTests extends ESIntegTestCase {
         }
 
 
-        termSuggest = termSuggestion("test")
-                .suggestMode("always") // Always, otherwise the results can vary between requests.
+        termSuggest = termSuggestion("text")
+                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
                 .text("ABCD")
-                .minWordLength(1)
-                .field("text");
+                .minWordLength(1);
         logger.info("--> run suggestions with four indices");
         try {
-            searchSuggest( termSuggest);
+            searchSuggest("test", termSuggest);
             fail(" can not suggest across multiple indices with different analysis chains");
         } catch (ReduceSearchPhaseException ex) {
             assertThat(ex.getCause(), instanceOf(IllegalStateException.class));
@@ -214,17 +213,27 @@ public class SuggestSearchTests extends ESIntegTestCase {
         refresh();
 
         DirectCandidateGeneratorBuilder generator = candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always").maxEdits(2);
-        PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("did_you_mean").field("name.shingled")
+        PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("name.shingled")
                 .addCandidateGenerator(generator)
                 .gramSize(3);
-        Suggest searchSuggest = searchSuggest( "ice tea", phraseSuggestion);
+        Suggest searchSuggest = searchSuggest("ice tea", "did_you_mean", phraseSuggestion);
         assertSuggestion(searchSuggest, 0, "did_you_mean", "iced tea");
 
         generator.suggestMode(null);
-        searchSuggest = searchSuggest( "ice tea", phraseSuggestion);
+        searchSuggest = searchSuggest( "ice tea", "did_you_mean", phraseSuggestion);
         assertSuggestionSize(searchSuggest, 0, 0, "did_you_mean");
     }
 
+    /**
+     * Creates a new {@link DirectCandidateGeneratorBuilder}
+     *
+     * @param field
+     *            the field this candidate generator operates on.
+     */
+    private DirectCandidateGeneratorBuilder candidateGenerator(String field) {
+        return new DirectCandidateGeneratorBuilder(field);
+    }
+
     // see #2729
     public void testSizeOneShard() throws Exception {
         prepareCreate("test").setSettings(
@@ -240,16 +249,15 @@ public class SuggestSearchTests extends ESIntegTestCase {
         SearchResponse search = client().prepareSearch().setQuery(matchQuery("text", "spellchecker")).get();
         assertThat("didn't ask for suggestions but got some", search.getSuggest(), nullValue());
 
-        TermSuggestionBuilder termSuggestion = termSuggestion("test")
-                .suggestMode("always") // Always, otherwise the results can vary between requests.
+        TermSuggestionBuilder termSuggestion = termSuggestion("text")
+                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
                 .text("abcd")
-                .field("text")
                 .size(10);
-        Suggest suggest = searchSuggest( termSuggestion);
+        Suggest suggest = searchSuggest("test", termSuggestion);
         assertSuggestion(suggest, 0, "test", 10, "abc0");
 
         termSuggestion.text("abcd").shardSize(5);
-        suggest = searchSuggest( termSuggestion);
+        suggest = searchSuggest("test", termSuggestion);
         assertSuggestion(suggest, 0, "test", 5, "abc0");
     }
 
@@ -283,21 +291,23 @@ public class SuggestSearchTests extends ESIntegTestCase {
         client().prepareIndex("test", "type1").setSource("name", "I like ice cream."));
         refresh();
 
-        PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("did_you_mean").field("name.shingled")
-                .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always").maxEdits(2))
+        PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("name.shingled")
+                .addCandidateGenerator(candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always").maxEdits(2))
                 .gramSize(3);
-        Suggest searchSuggest = searchSuggest( "ice tea", phraseSuggestion);
+        Suggest searchSuggest = searchSuggest("ice tea", "did_you_mean", phraseSuggestion);
         assertSuggestion(searchSuggest, 0, 0, "did_you_mean", "iced tea");
 
-        phraseSuggestion.field("nosuchField");
+        phraseSuggestion = phraseSuggestion("nosuchField")
+                .addCandidateGenerator(candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always").maxEdits(2))
+                .gramSize(3);
         {
             SearchRequestBuilder searchBuilder = client().prepareSearch().setSize(0);
-            searchBuilder.suggest(new SuggestBuilder().setText("tetsting sugestion").addSuggestion(phraseSuggestion));
+            searchBuilder.suggest(new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean", phraseSuggestion));
             assertThrows(searchBuilder, SearchPhaseExecutionException.class);
         }
         {
             SearchRequestBuilder searchBuilder = client().prepareSearch().setSize(0);
-            searchBuilder.suggest(new SuggestBuilder().setText("tetsting sugestion").addSuggestion(phraseSuggestion));
+            searchBuilder.suggest(new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean", phraseSuggestion));
             assertThrows(searchBuilder, SearchPhaseExecutionException.class);
         }
     }
@@ -315,15 +325,14 @@ public class SuggestSearchTests extends ESIntegTestCase {
         SearchResponse search = client().prepareSearch().setQuery(matchQuery("text", "spellcecker")).get();
         assertThat("didn't ask for suggestions but got some", search.getSuggest(), nullValue());
 
-        TermSuggestionBuilder termSuggest = termSuggestion("test")
-                .suggestMode("always") // Always, otherwise the results can vary between requests.
-                .text("abcd")
-                .field("text");
-        Suggest suggest = searchSuggest( termSuggest);
+        TermSuggestionBuilder termSuggest = termSuggestion("text")
+                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
+                .text("abcd");
+        Suggest suggest = searchSuggest("test", termSuggest);
         assertSuggestion(suggest, 0, "test", "aacd", "abbd", "abcc");
         assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
 
-        suggest = searchSuggest( termSuggest);
+        suggest = searchSuggest("test", termSuggest);
         assertSuggestion(suggest, 0, "test", "aacd","abbd", "abcc");
         assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
     }
@@ -332,18 +341,17 @@ public class SuggestSearchTests extends ESIntegTestCase {
         createIndex("test");
         ensureGreen();
 
-        index("test", "type1", "1", "foo", "bar");
+        index("test", "type1", "1", "text", "bar");
         refresh();
 
-        TermSuggestionBuilder termSuggest = termSuggestion("test")
-                .suggestMode("always") // Always, otherwise the results can vary between requests.
-                .text("abcd")
-                .field("text");
-        Suggest suggest = searchSuggest( termSuggest);
+        TermSuggestionBuilder termSuggest = termSuggestion("text")
+                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
+                .text("abcd");
+        Suggest suggest = searchSuggest("test", termSuggest);
         assertSuggestionSize(suggest, 0, 0, "test");
         assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
 
-        suggest = searchSuggest( termSuggest);
+        suggest = searchSuggest("test", termSuggest);
         assertSuggestionSize(suggest, 0, 0, "test");
         assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
     }
@@ -358,16 +366,17 @@ public class SuggestSearchTests extends ESIntegTestCase {
         index("test", "typ1", "4", "field1", "prefix_abcc", "field2", "prefix_eggg");
         refresh();
 
-        Suggest suggest = searchSuggest(
-                termSuggestion("size1")
-                        .size(1).text("prefix_abcd").maxTermFreq(10).prefixLength(1).minDocFreq(0)
-                        .field("field1").suggestMode("always"),
-                termSuggestion("field2")
-                        .field("field2").text("prefix_eeeh prefix_efgh")
-                        .maxTermFreq(10).minDocFreq(0).suggestMode("always"),
-                termSuggestion("accuracy")
-                        .field("field2").text("prefix_efgh").setAccuracy(1f)
-                        .maxTermFreq(10).minDocFreq(0).suggestMode("always"));
+        Map<String, SuggestionBuilder<?>> suggestions = new HashMap<>();
+        suggestions.put("size1", termSuggestion("field1")
+                .size(1).text("prefix_abcd").maxTermFreq(10).prefixLength(1).minDocFreq(0)
+                .suggestMode(SuggestMode.ALWAYS));
+        suggestions.put("field2", termSuggestion("field2")
+                .text("prefix_eeeh prefix_efgh")
+                .maxTermFreq(10).minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
+        suggestions.put("accuracy", termSuggestion("field2")
+                .text("prefix_efgh").accuracy(1f)
+                .maxTermFreq(10).minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
+        Suggest suggest = searchSuggest(null, 0, suggestions);
         assertSuggestion(suggest, 0, "size1", "prefix_aacd");
         assertThat(suggest.getSuggestion("field2").getEntries().get(0).getText().string(), equalTo("prefix_eeeh"));
         assertSuggestion(suggest, 0, "field2", "prefix_efgh");
@@ -401,17 +410,18 @@ public class SuggestSearchTests extends ESIntegTestCase {
         }
         refresh();
 
-        Suggest suggest = searchSuggest( "prefix_abcd",
-                termSuggestion("size3SortScoreFirst")
-                        .size(3).minDocFreq(0).field("field1").suggestMode("always"),
-                termSuggestion("size10SortScoreFirst")
-                        .size(10).minDocFreq(0).field("field1").suggestMode("always").shardSize(50),
-                termSuggestion("size3SortScoreFirstMaxEdits1")
-                        .maxEdits(1)
-                        .size(10).minDocFreq(0).field("field1").suggestMode("always"),
-                termSuggestion("size10SortFrequencyFirst")
-                        .size(10).sort("frequency").shardSize(1000)
-                        .minDocFreq(0).field("field1").suggestMode("always"));
+        Map<String, SuggestionBuilder<?>> suggestions = new HashMap<>();
+        suggestions.put("size3SortScoreFirst", termSuggestion("field1")
+                .size(3).minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
+        suggestions.put("size10SortScoreFirst", termSuggestion("field1")
+                .size(10).minDocFreq(0).suggestMode(SuggestMode.ALWAYS).shardSize(50));
+        suggestions.put("size3SortScoreFirstMaxEdits1", termSuggestion("field1")
+                .maxEdits(1)
+                .size(10).minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
+        suggestions.put("size10SortFrequencyFirst", termSuggestion("field1")
+                .size(10).sort(SortBy.FREQUENCY).shardSize(1000)
+                .minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
+        Suggest suggest = searchSuggest("prefix_abcd", 0, suggestions);
 
         // The commented out assertions fail sometimes because suggestions are based off of shard frequencies instead of index frequencies.
         assertSuggestion(suggest, 0, "size3SortScoreFirst", "prefix_aacd", "prefix_abcc", "prefix_accd");
@@ -435,9 +445,9 @@ public class SuggestSearchTests extends ESIntegTestCase {
         index("test", "typ1", "1", "body", "this is a test");
         refresh();
 
-        Suggest searchSuggest = searchSuggest( "a an the",
-                phraseSuggestion("simple_phrase").field("body").gramSize(1)
-                        .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").minWordLength(1).suggestMode("always"))
+        Suggest searchSuggest = searchSuggest( "a an the", "simple_phrase",
+                phraseSuggestion("body").gramSize(1)
+                        .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
                         .size(1));
         assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
     }
@@ -471,15 +481,15 @@ public class SuggestSearchTests extends ESIntegTestCase {
         index("test", "type1", "3", "body", "hello words");
         refresh();
 
-        Suggest searchSuggest = searchSuggest( "hello word",
-                phraseSuggestion("simple_phrase").field("body")
-                        .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").prefixLength(4).minWordLength(1).suggestMode("always"))
+        Suggest searchSuggest = searchSuggest( "hello word", "simple_phrase",
+                phraseSuggestion("body")
+                        .addCandidateGenerator(candidateGenerator("body").prefixLength(4).minWordLength(1).suggestMode("always"))
                         .size(1).confidence(1.0f));
         assertSuggestion(searchSuggest, 0, "simple_phrase", "hello words");
 
-        searchSuggest = searchSuggest( "hello word",
-                phraseSuggestion("simple_phrase").field("body")
-                        .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").prefixLength(2).minWordLength(1).suggestMode("always"))
+        searchSuggest = searchSuggest( "hello word", "simple_phrase",
+                phraseSuggestion("body")
+                        .addCandidateGenerator(candidateGenerator("body").prefixLength(2).minWordLength(1).suggestMode("always"))
                         .size(1).confidence(1.0f));
         assertSuggestion(searchSuggest, 0, "simple_phrase", "hello world");
     }
@@ -526,88 +536,87 @@ public class SuggestSearchTests extends ESIntegTestCase {
         }
         refresh();
 
-        PhraseSuggestionBuilder phraseSuggest = phraseSuggestion("simple_phrase")
-                .field("bigram").gramSize(2).analyzer("body")
+        PhraseSuggestionBuilder phraseSuggest = phraseSuggestion("bigram").gramSize(2).analyzer("body")
                 .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
                 .size(1);
-        Suggest searchSuggest = searchSuggest( "american ame", phraseSuggest);
+        Suggest searchSuggest = searchSuggest( "american ame", "simple_phrase", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "american ace");
         assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getText().string(), equalTo("american ame"));
 
         phraseSuggest.realWordErrorLikelihood(0.95f);
-        searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
+        searchSuggest = searchSuggest( "Xor the Got-Jewel", "simple_phrase", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
         // Check the "text" field this one time.
         assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getText().string(), equalTo("Xor the Got-Jewel"));
 
         // Ask for highlighting
         phraseSuggest.highlight("<em>", "</em>");
-        searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
+        searchSuggest = searchSuggest( "Xor the Got-Jewel", "simple_phrase", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
         assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getOptions().get(0).getHighlighted().string(), equalTo("<em>xorr</em> the <em>god</em> jewel"));
 
         // pass in a correct phrase
         phraseSuggest.highlight(null, null).confidence(0f).size(1).maxErrors(0.5f);
-        searchSuggest = searchSuggest( "Xorr the God-Jewel", phraseSuggest);
+        searchSuggest = searchSuggest( "Xorr the God-Jewel", "simple_phrase", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
 
         // pass in a correct phrase - set confidence to 2
         phraseSuggest.confidence(2f);
-        searchSuggest = searchSuggest( "Xorr the God-Jewel", phraseSuggest);
+        searchSuggest = searchSuggest( "Xorr the God-Jewel", "simple_phrase", phraseSuggest);
         assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
 
         // pass in a correct phrase - set confidence to 0.99
         phraseSuggest.confidence(0.99f);
-        searchSuggest = searchSuggest( "Xorr the God-Jewel", phraseSuggest);
+        searchSuggest = searchSuggest( "Xorr the God-Jewel", "simple_phrase", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
 
         //test reverse suggestions with pre & post filter
         phraseSuggest
             .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
             .addCandidateGenerator(candidateGenerator("body_reverse").minWordLength(1).suggestMode("always").preFilter("reverse").postFilter("reverse"));
-        searchSuggest = searchSuggest( "xor the yod-Jewel", phraseSuggest);
+        searchSuggest = searchSuggest( "xor the yod-Jewel",  "simple_phrase", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
 
         // set all mass to trigrams (not indexed)
         phraseSuggest.clearCandidateGenerators()
             .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
-            .smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(1,0,0));
-        searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
+            .smoothingModel(new LinearInterpolation(1,0,0));
+        searchSuggest = searchSuggest( "Xor the Got-Jewel", "simple_phrase", phraseSuggest);
         assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
 
         // set all mass to bigrams
-        phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(0,1,0));
-        searchSuggest =  searchSuggest( "Xor the Got-Jewel", phraseSuggest);
+        phraseSuggest.smoothingModel(new LinearInterpolation(0,1,0));
+        searchSuggest =  searchSuggest( "Xor the Got-Jewel", "simple_phrase", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
 
         // distribute mass
-        phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(0.4,0.4,0.2));
-        searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
+        phraseSuggest.smoothingModel(new LinearInterpolation(0.4,0.4,0.2));
+        searchSuggest = searchSuggest( "Xor the Got-Jewel", "simple_phrase", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
 
-        searchSuggest = searchSuggest( "american ame", phraseSuggest);
+        searchSuggest = searchSuggest( "american ame", "simple_phrase", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "american ace");
 
         // try all smoothing methods
-        phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(0.4,0.4,0.2));
-        searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
+        phraseSuggest.smoothingModel(new LinearInterpolation(0.4,0.4,0.2));
+        searchSuggest = searchSuggest( "Xor the Got-Jewel", "simple_phrase", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
 
-        phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.Laplace(0.2));
-        searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
+        phraseSuggest.smoothingModel(new Laplace(0.2));
+        searchSuggest = searchSuggest( "Xor the Got-Jewel",  "simple_phrase", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
 
-        phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.StupidBackoff(0.1));
-        searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
+        phraseSuggest.smoothingModel(new StupidBackoff(0.1));
+        searchSuggest = searchSuggest( "Xor the Got-Jewel",  "simple_phrase",phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
 
         // check tokenLimit
         phraseSuggest.smoothingModel(null).tokenLimit(4);
-        searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
+        searchSuggest = searchSuggest( "Xor the Got-Jewel", "simple_phrase", phraseSuggest);
         assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
 
-        phraseSuggest.tokenLimit(15).smoothingModel(new PhraseSuggestionBuilder.StupidBackoff(0.1));
-        searchSuggest = searchSuggest( "Xor the Got-Jewel Xor the Got-Jewel Xor the Got-Jewel", phraseSuggest);
+        phraseSuggest.tokenLimit(15).smoothingModel(new StupidBackoff(0.1));
+        searchSuggest = searchSuggest( "Xor the Got-Jewel Xor the Got-Jewel Xor the Got-Jewel", "simple_phrase", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel xorr the god jewel xorr the god jewel");
         // Check the name this time because we're repeating it which is funky
         assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getText().string(), equalTo("Xor the Got-Jewel Xor the Got-Jewel Xor the Got-Jewel"));
@@ -663,22 +672,21 @@ public class SuggestSearchTests extends ESIntegTestCase {
         index("test", "type1", "2", "body", line, "body_reverse", line, "bigram", line);
         refresh();
 
-        PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("simple_phrase")
+        PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("bigram")
                 .realWordErrorLikelihood(0.95f)
-                .field("bigram")
                 .gramSize(2)
                 .analyzer("body")
                 .addCandidateGenerator(candidateGenerator("body").minWordLength(1).prefixLength(1).suggestMode("always").size(1).accuracy(0.1f))
-                .smoothingModel(new PhraseSuggestionBuilder.StupidBackoff(0.1))
+                .smoothingModel(new StupidBackoff(0.1))
                 .maxErrors(1.0f)
                 .size(5);
-        Suggest searchSuggest = searchSuggest( "Xorr the Gut-Jewel", phraseSuggestion);
+        Suggest searchSuggest = searchSuggest("Xorr the Gut-Jewel", "simple_phrase", phraseSuggestion);
         assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
 
         // we allow a size of 2 now on the shard generator level so "god" will be found since it's LD2
         phraseSuggestion.clearCandidateGenerators()
                 .addCandidateGenerator(candidateGenerator("body").minWordLength(1).prefixLength(1).suggestMode("always").size(2).accuracy(0.1f));
-        searchSuggest = searchSuggest( "Xorr the Gut-Jewel", phraseSuggestion);
+        searchSuggest = searchSuggest( "Xorr the Gut-Jewel", "simple_phrase", phraseSuggestion);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
     }
 
@@ -723,51 +731,56 @@ public class SuggestSearchTests extends ESIntegTestCase {
         NumShards numShards = getNumShards("test");
 
         // Lets make sure some things throw exceptions
-        PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("simple_phrase")
-                .field("bigram")
+        PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("bigram")
                 .analyzer("body")
                 .addCandidateGenerator(candidateGenerator("does_not_exist").minWordLength(1).suggestMode("always"))
                 .realWordErrorLikelihood(0.95f)
                 .maxErrors(0.5f)
                 .size(1);
+        Map<String, SuggestionBuilder<?>> suggestion = new HashMap<>();
+        suggestion.put("simple_phrase", phraseSuggestion);
         try {
-            searchSuggest( "Xor the Got-Jewel", numShards.numPrimaries, phraseSuggestion);
+            searchSuggest("Xor the Got-Jewel", numShards.numPrimaries, suggestion);
             fail("field does not exists");
         } catch (SearchPhaseExecutionException e) {}
 
         phraseSuggestion.clearCandidateGenerators().analyzer(null);
         try {
-            searchSuggest( "Xor the Got-Jewel", numShards.numPrimaries, phraseSuggestion);
+            searchSuggest("Xor the Got-Jewel", numShards.numPrimaries, suggestion);
             fail("analyzer does only produce ngrams");
         } catch (SearchPhaseExecutionException e) {
         }
 
         phraseSuggestion.analyzer("bigram");
         try {
-            searchSuggest( "Xor the Got-Jewel", numShards.numPrimaries, phraseSuggestion);
+            searchSuggest("Xor the Got-Jewel", numShards.numPrimaries, suggestion);
             fail("analyzer does only produce ngrams");
         } catch (SearchPhaseExecutionException e) {
         }
 
         // Now we'll make sure some things don't
         phraseSuggestion.forceUnigrams(false);
-        searchSuggest( "Xor the Got-Jewel", phraseSuggestion);
+        searchSuggest( "Xor the Got-Jewel", 0, suggestion);
 
         // Field doesn't produce unigrams but the analyzer does
-        phraseSuggestion.forceUnigrams(true).field("bigram").analyzer("ngram");
-        searchSuggest( "Xor the Got-Jewel",
-                phraseSuggestion);
+        phraseSuggestion.forceUnigrams(true).analyzer("ngram");
+        searchSuggest( "Xor the Got-Jewel", 0, suggestion);
 
-        phraseSuggestion.field("ngram").analyzer("myDefAnalyzer")
+        phraseSuggestion = phraseSuggestion("ngram")
+                .analyzer("myDefAnalyzer")
+                .forceUnigrams(true)
+                .realWordErrorLikelihood(0.95f)
+                .maxErrors(0.5f)
+                .size(1)
                 .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"));
-        Suggest suggest = searchSuggest( "Xor the Got-Jewel", phraseSuggestion);
+        Suggest suggest = searchSuggest( "Xor the Got-Jewel", 0, suggestion);
 
         // "xorr the god jewel" and and "xorn the god jewel" have identical scores (we are only using unigrams to score), so we tie break by
         // earlier term (xorn):
         assertSuggestion(suggest, 0, "simple_phrase", "xorn the god jewel");
 
         phraseSuggestion.analyzer(null);
-        suggest = searchSuggest( "Xor the Got-Jewel", phraseSuggestion);
+        suggest = searchSuggest( "Xor the Got-Jewel", 0, suggestion);
 
         // In this case xorr has a better score than xorn because we set the field back to the default (my_shingle2) analyzer, so the
         // probability that the term is not in the dictionary but is NOT a misspelling is relatively high in this case compared to the
@@ -782,9 +795,9 @@ public class SuggestSearchTests extends ESIntegTestCase {
                 client().prepareIndex("test", "type1", "2").setSource("field1", "foobar2").setRouting("2"),
                 client().prepareIndex("test", "type1", "3").setSource("field1", "foobar3").setRouting("3"));
 
-        Suggest suggest = searchSuggest( "foobar",
-                termSuggestion("simple")
-                        .size(10).minDocFreq(0).field("field1").suggestMode("always"));
+        Suggest suggest = searchSuggest( "foobar", "simple",
+                termSuggestion("field1")
+                        .size(10).minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
         ElasticsearchAssertions.assertSuggestionSize(suggest, 0, 3, "simple");
     }
 
@@ -823,15 +836,15 @@ public class SuggestSearchTests extends ESIntegTestCase {
         // When searching on a shard with a non existing mapping, we should fail
         SearchRequestBuilder request = client().prepareSearch().setSize(0)
                 .suggest(
-                        new SuggestBuilder().setText("tetsting sugestion").addSuggestion(
-                                phraseSuggestion("did_you_mean").field("fielddoesnotexist").maxErrors(5.0f)));
+                        new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean",
+                                phraseSuggestion("fielddoesnotexist").maxErrors(5.0f)));
         assertThrows(request, SearchPhaseExecutionException.class);
 
         // When searching on a shard which does not hold yet any document of an existing type, we should not fail
         SearchResponse searchResponse = client().prepareSearch().setSize(0)
                 .suggest(
-                        new SuggestBuilder().setText("tetsting sugestion").addSuggestion(
-                                phraseSuggestion("did_you_mean").field("name").maxErrors(5.0f)))
+                        new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean",
+                                phraseSuggestion("name").maxErrors(5.0f)))
             .get();
         ElasticsearchAssertions.assertNoFailures(searchResponse);
         ElasticsearchAssertions.assertSuggestion(searchResponse.getSuggest(), 0, 0, "did_you_mean", "testing suggestions");
@@ -869,8 +882,8 @@ public class SuggestSearchTests extends ESIntegTestCase {
         SearchResponse searchResponse = client().prepareSearch()
                 .setSize(0)
                 .suggest(
-                        new SuggestBuilder().setText("tetsting sugestion").addSuggestion(
-                                phraseSuggestion("did_you_mean").field("name").maxErrors(5.0f)))
+                        new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean",
+                                phraseSuggestion("name").maxErrors(5.0f)))
                 .get();
 
         assertNoFailures(searchResponse);
@@ -927,17 +940,15 @@ public class SuggestSearchTests extends ESIntegTestCase {
         }
         refresh();
 
-        Suggest searchSuggest = searchSuggest("nobel prize", phraseSuggestion("simple_phrase")
-                .field("body")
-                .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
+        Suggest searchSuggest = searchSuggest("nobel prize", "simple_phrase", phraseSuggestion("body")
+                .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
                 .confidence(2f)
                 .maxErrors(5f)
                 .size(1));
         assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
 
-        searchSuggest = searchSuggest("noble prize", phraseSuggestion("simple_phrase")
-                .field("body")
-                .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
+        searchSuggest = searchSuggest("noble prize", "simple_phrase", phraseSuggestion("body")
+                .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
                 .confidence(2f)
                 .maxErrors(5f)
                 .size(1));
@@ -1067,8 +1078,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
         indexRandom(true, builders);
 
         PhraseSuggestionBuilder suggest = phraseSuggestion("title")
-                .field("title")
-                .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("title")
+                .addCandidateGenerator(candidateGenerator("title")
                         .suggestMode("always")
                         .maxTermFreq(.99f)
                         .size(1000) // Setting a silly high size helps of generate a larger list of candidates for testing.
@@ -1078,13 +1088,13 @@ public class SuggestSearchTests extends ESIntegTestCase {
                 .maxErrors(2f)
                 .shardSize(30000)
                 .size(30000);
-        Suggest searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", suggest);
+        Suggest searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", suggest);
         assertSuggestion(searchSuggest, 0, 0, "title", "united states house of representatives elections in washington 2006");
         assertSuggestionSize(searchSuggest, 0, 25480, "title");  // Just to prove that we've run through a ton of options
 
         suggest.size(1);
         long start = System.currentTimeMillis();
-        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", suggest);
+        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", suggest);
         long total = System.currentTimeMillis() - start;
         assertSuggestion(searchSuggest, 0, 0, "title", "united states house of representatives elections in washington 2006");
         // assertThat(total, lessThan(1000L)); // Takes many seconds without fix - just for debugging
@@ -1132,8 +1142,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
 
         // suggest without collate
         PhraseSuggestionBuilder suggest = phraseSuggestion("title")
-                .field("title")
-                .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("title")
+                .addCandidateGenerator(candidateGenerator("title")
                         .suggestMode("always")
                         .maxTermFreq(.99f)
                         .size(10)
@@ -1143,7 +1152,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
                 .maxErrors(2f)
                 .shardSize(30000)
                 .size(10);
-        Suggest searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", suggest);
+        Suggest searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", suggest);
         assertSuggestionSize(searchSuggest, 0, 10, "title");
 
         // suggest with collate
@@ -1156,11 +1165,11 @@ public class SuggestSearchTests extends ESIntegTestCase {
                 .string();
         PhraseSuggestionBuilder filteredQuerySuggest = suggest.collateQuery(filterString);
         filteredQuerySuggest.collateParams(Collections.singletonMap("field", "title"));
-        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", filteredQuerySuggest);
+        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", filteredQuerySuggest);
         assertSuggestionSize(searchSuggest, 0, 2, "title");
 
         // collate suggest with no result (boundary case)
-        searchSuggest = searchSuggest("Elections of Representatives Parliament", filteredQuerySuggest);
+        searchSuggest = searchSuggest("Elections of Representatives Parliament", "title", filteredQuerySuggest);
         assertSuggestionSize(searchSuggest, 0, 0, "title");
 
         NumShards numShards = getNumShards("test");
@@ -1174,8 +1183,10 @@ public class SuggestSearchTests extends ESIntegTestCase {
                 .endObject()
                 .string();
         PhraseSuggestionBuilder incorrectFilteredSuggest = suggest.collateQuery(incorrectFilterString);
+        Map<String, SuggestionBuilder<?>> namedSuggestion = new HashMap<>();
+        namedSuggestion.put("my_title_suggestion", incorrectFilteredSuggest);
         try {
-            searchSuggest("united states house of representatives elections in washington 2006", numShards.numPrimaries, incorrectFilteredSuggest);
+            searchSuggest("united states house of representatives elections in washington 2006", numShards.numPrimaries, namedSuggestion);
             fail("Post query error has been swallowed");
         } catch(ElasticsearchException e) {
             // expected
@@ -1191,7 +1202,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
                 .string();
 
         PhraseSuggestionBuilder filteredFilterSuggest = suggest.collateQuery(filterStringAsFilter);
-        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", filteredFilterSuggest);
+        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", filteredFilterSuggest);
         assertSuggestionSize(searchSuggest, 0, 2, "title");
 
         // collate suggest with bad query
@@ -1205,7 +1216,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
 
         PhraseSuggestionBuilder in = suggest.collateQuery(filterStr);
         try {
-            searchSuggest("united states house of representatives elections in washington 2006", numShards.numPrimaries, in);
+            searchSuggest("united states house of representatives elections in washington 2006", numShards.numPrimaries, namedSuggestion);
             fail("Post filter error has been swallowed");
         } catch(ElasticsearchException e) {
             //expected
@@ -1223,7 +1234,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
 
         PhraseSuggestionBuilder phraseSuggestWithNoParams = suggest.collateQuery(collateWithParams);
         try {
-            searchSuggest("united states house of representatives elections in washington 2006", numShards.numPrimaries, phraseSuggestWithNoParams);
+            searchSuggest("united states house of representatives elections in washington 2006", numShards.numPrimaries, namedSuggestion);
             fail("Malformed query (lack of additional params) should fail");
         } catch (ElasticsearchException e) {
             // expected
@@ -1235,53 +1246,38 @@ public class SuggestSearchTests extends ESIntegTestCase {
         params.put("query_field", "title");
 
         PhraseSuggestionBuilder phraseSuggestWithParams = suggest.collateQuery(collateWithParams).collateParams(params);
-        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", phraseSuggestWithParams);
+        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", phraseSuggestWithParams);
         assertSuggestionSize(searchSuggest, 0, 2, "title");
 
         // collate query request with prune set to true
         PhraseSuggestionBuilder phraseSuggestWithParamsAndReturn = suggest.collateQuery(collateWithParams).collateParams(params).collatePrune(true);
-        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", phraseSuggestWithParamsAndReturn);
+        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", phraseSuggestWithParamsAndReturn);
         assertSuggestionSize(searchSuggest, 0, 10, "title");
         assertSuggestionPhraseCollateMatchExists(searchSuggest, "title", 2);
     }
 
-    protected Suggest searchSuggest(SuggestionBuilder<?>... suggestion) {
-        return searchSuggest(null, suggestion);
+    protected Suggest searchSuggest(String name, SuggestionBuilder<?> suggestion) {
+        return searchSuggest(null, name, suggestion);
     }
 
-    protected Suggest searchSuggest(String suggestText, SuggestionBuilder<?>... suggestions) {
-        return searchSuggest(suggestText, 0, suggestions);
+    protected Suggest searchSuggest(String suggestText, String name, SuggestionBuilder<?> suggestion) {
+        Map<String, SuggestionBuilder<?>> map = new HashMap<>();
+        map.put(name, suggestion);
+        return searchSuggest(suggestText, 0, map);
     }
 
-    protected Suggest searchSuggest(String suggestText, int expectShardsFailed, SuggestionBuilder<?>... suggestions) {
-        if (randomBoolean()) {
-            SearchRequestBuilder builder = client().prepareSearch().setSize(0);
-            SuggestBuilder suggestBuilder = new SuggestBuilder();
-            if (suggestText != null) {
-                suggestBuilder.setText(suggestText);
-            }
-            for (SuggestionBuilder<?> suggestion : suggestions) {
-                suggestBuilder.addSuggestion(suggestion);
-            }
-            builder.suggest(suggestBuilder);
-            SearchResponse actionGet = builder.execute().actionGet();
-            assertThat(Arrays.toString(actionGet.getShardFailures()), actionGet.getFailedShards(), equalTo(expectShardsFailed));
-            return actionGet.getSuggest();
-        } else {
-            SuggestRequestBuilder builder = client().prepareSuggest();
-            if (suggestText != null) {
-                builder.setSuggestText(suggestText);
-            }
-            for (SuggestionBuilder<?> suggestion : suggestions) {
-                builder.addSuggestion(suggestion);
-            }
-
-            SuggestResponse actionGet = builder.execute().actionGet();
-            assertThat(Arrays.toString(actionGet.getShardFailures()), actionGet.getFailedShards(), equalTo(expectShardsFailed));
-            if (expectShardsFailed > 0) {
-                throw new SearchPhaseExecutionException("suggest", "Suggest execution failed", new ShardSearchFailure[0]);
-            }
-            return actionGet.getSuggest();
+    protected Suggest searchSuggest(String suggestText, int expectShardsFailed, Map<String, SuggestionBuilder<?>> suggestions) {
+        SearchRequestBuilder builder = client().prepareSearch().setSize(0);
+        SuggestBuilder suggestBuilder = new SuggestBuilder();
+        if (suggestText != null) {
+            suggestBuilder.setGlobalText(suggestText);
         }
+        for (Entry<String, SuggestionBuilder<?>> suggestion : suggestions.entrySet()) {
+            suggestBuilder.addSuggestion(suggestion.getKey(), suggestion.getValue());
+        }
+        builder.suggest(suggestBuilder);
+        SearchResponse actionGet = builder.execute().actionGet();
+        assertThat(Arrays.toString(actionGet.getShardFailures()), actionGet.getFailedShards(), equalTo(expectShardsFailed));
+        return actionGet.getSuggest();
     }
 }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
index 78c40088575..c0ad3782bf9 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
@@ -23,8 +23,8 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Accountable;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Injector;
@@ -153,7 +153,7 @@ public class TemplateQueryParserTests extends ESTestCase {
             }
         });
         IndicesQueriesRegistry indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
-        context = new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry);
+        context = new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry, null);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
index 33ab695d527..090667b4543 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
@@ -19,21 +19,10 @@
 
 package org.elasticsearch.painless;
 
-import org.antlr.v4.runtime.ParserRuleContext;
-import org.elasticsearch.painless.Definition.Cast;
-import org.elasticsearch.painless.Definition.Constructor;
-import org.elasticsearch.painless.Definition.Field;
-import org.elasticsearch.painless.Definition.Method;
-import org.elasticsearch.painless.Definition.Pair;
-import org.elasticsearch.painless.Definition.Sort;
-import org.elasticsearch.painless.Definition.Struct;
-import org.elasticsearch.painless.Definition.Transform;
-import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.PainlessParser.AfterthoughtContext;
 import org.elasticsearch.painless.PainlessParser.ArgumentsContext;
 import org.elasticsearch.painless.PainlessParser.AssignmentContext;
 import org.elasticsearch.painless.PainlessParser.BinaryContext;
-import org.elasticsearch.painless.PainlessParser.BlockContext;
 import org.elasticsearch.painless.PainlessParser.BoolContext;
 import org.elasticsearch.painless.PainlessParser.BreakContext;
 import org.elasticsearch.painless.PainlessParser.CastContext;
@@ -47,8 +36,8 @@ import org.elasticsearch.painless.PainlessParser.DecltypeContext;
 import org.elasticsearch.painless.PainlessParser.DeclvarContext;
 import org.elasticsearch.painless.PainlessParser.DoContext;
 import org.elasticsearch.painless.PainlessParser.EmptyContext;
+import org.elasticsearch.painless.PainlessParser.EmptyscopeContext;
 import org.elasticsearch.painless.PainlessParser.ExprContext;
-import org.elasticsearch.painless.PainlessParser.ExpressionContext;
 import org.elasticsearch.painless.PainlessParser.ExtbraceContext;
 import org.elasticsearch.painless.PainlessParser.ExtcallContext;
 import org.elasticsearch.painless.PainlessParser.ExtcastContext;
@@ -75,7 +64,6 @@ import org.elasticsearch.painless.PainlessParser.PreincContext;
 import org.elasticsearch.painless.PainlessParser.ReturnContext;
 import org.elasticsearch.painless.PainlessParser.SingleContext;
 import org.elasticsearch.painless.PainlessParser.SourceContext;
-import org.elasticsearch.painless.PainlessParser.StatementContext;
 import org.elasticsearch.painless.PainlessParser.ThrowContext;
 import org.elasticsearch.painless.PainlessParser.TrapContext;
 import org.elasticsearch.painless.PainlessParser.TrueContext;
@@ -83,3077 +71,384 @@ import org.elasticsearch.painless.PainlessParser.TryContext;
 import org.elasticsearch.painless.PainlessParser.UnaryContext;
 import org.elasticsearch.painless.PainlessParser.WhileContext;
 
-import java.util.ArrayDeque;
-import java.util.Arrays;
-import java.util.Deque;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import static org.elasticsearch.painless.PainlessParser.ADD;
-import static org.elasticsearch.painless.PainlessParser.BWAND;
-import static org.elasticsearch.painless.PainlessParser.BWOR;
-import static org.elasticsearch.painless.PainlessParser.BWXOR;
-import static org.elasticsearch.painless.PainlessParser.DIV;
-import static org.elasticsearch.painless.PainlessParser.LSH;
-import static org.elasticsearch.painless.PainlessParser.MUL;
-import static org.elasticsearch.painless.PainlessParser.REM;
-import static org.elasticsearch.painless.PainlessParser.RSH;
-import static org.elasticsearch.painless.PainlessParser.SUB;
-import static org.elasticsearch.painless.PainlessParser.USH;
-
 class Analyzer extends PainlessParserBaseVisitor<Void> {
-    private static class Variable {
-        final String name;
-        final Type type;
-        final int slot;
-
-        private Variable(final String name, final Type type, final int slot) {
-            this.name = name;
-            this.type = type;
-            this.slot = slot;
-        }
-    }
-
     static void analyze(final Metadata metadata) {
         new Analyzer(metadata);
     }
 
-    private final Metadata metadata;
-    private final Definition definition;
-    private final CompilerSettings settings;
-
-    private final Deque<Integer> scopes = new ArrayDeque<>();
-    private final Deque<Variable> variables = new ArrayDeque<>();
+    private final AnalyzerStatement statement;
+    private final AnalyzerExpression expression;
+    private final AnalyzerExternal external;
 
     private Analyzer(final Metadata metadata) {
-        this.metadata = metadata;
-        definition = metadata.definition;
-        settings = metadata.settings;
+        final Definition definition = metadata.definition;
 
-        incrementScope();
-        addVariable(null, "#this", definition.execType);
-        metadata.inputValueSlot = addVariable(null, "input", definition.smapType).slot;
-        metadata.scoreValueSlot = addVariable(null, "_score", definition.floatType).slot;
-        metadata.loopCounterSlot = addVariable(null, "#loop", definition.intType).slot;
+        final AnalyzerUtility utility = new AnalyzerUtility();
+        final AnalyzerCaster caster = new AnalyzerCaster(definition);
+        final AnalyzerPromoter promoter = new AnalyzerPromoter(definition);
+
+        statement = new AnalyzerStatement(metadata, this, utility, caster);
+        expression = new AnalyzerExpression(metadata, this, caster, promoter);
+        external = new AnalyzerExternal(metadata, this, utility, caster, promoter);
+
+        utility.incrementScope();
+        utility.addVariable(null, "#this", definition.execType);
+        metadata.inputValueSlot = utility.addVariable(null, "input", definition.smapType).slot;
+        metadata.scoreValueSlot = utility.addVariable(null, "_score", definition.floatType).slot;
+        metadata.loopCounterSlot = utility.addVariable(null, "#loop", definition.intType).slot;
 
         metadata.createStatementMetadata(metadata.root);
         visit(metadata.root);
 
-        decrementScope();
-    }
-
-    void incrementScope() {
-        scopes.push(0);
-    }
-
-    void decrementScope() {
-        int remove = scopes.pop();
-
-        while (remove > 0) {
-            variables.pop();
-            --remove;
-        }
-    }
-
-    Variable getVariable(final String name) {
-        final Iterator<Variable> itr = variables.iterator();
-
-        while (itr.hasNext()) {
-            final Variable variable = itr.next();
-
-            if (variable.name.equals(name)) {
-                return variable;
-            }
-        }
-
-        return null;
-    }
-
-    Variable addVariable(final ParserRuleContext source, final String name, final Type type) {
-        if (getVariable(name) != null) {
-            if (source == null) {
-                throw new IllegalArgumentException("Argument name [" + name + "] already defined within the scope.");
-            } else {
-                throw new IllegalArgumentException(
-                    Metadata.error(source) + "Variable name [" + name + "] already defined within the scope.");
-            }
-        }
-
-        final Variable previous = variables.peekFirst();
-        int slot = 0;
-
-        if (previous != null) {
-            slot += previous.slot + previous.type.type.getSize();
-        }
-
-        final Variable variable = new Variable(name, type, slot);
-        variables.push(variable);
-
-        final int update = scopes.pop() + 1;
-        scopes.push(update);
-
-        return variable;
+        utility.decrementScope();
     }
 
     @Override
     public Void visitSource(final SourceContext ctx) {
-        final Metadata.StatementMetadata sourcesmd = metadata.getStatementMetadata(ctx);
-        final List<StatementContext> statectxs = ctx.statement();
-        final StatementContext lastctx = statectxs.get(statectxs.size() - 1);
-
-        incrementScope();
-
-        for (final StatementContext statectx : statectxs) {
-            if (sourcesmd.allLast) {
-                throw new IllegalArgumentException(Metadata.error(statectx) +
-                    "Statement will never be executed because all prior paths escape.");
-            }
-
-            final Metadata.StatementMetadata statesmd = metadata.createStatementMetadata(statectx);
-            statesmd.lastSource = statectx == lastctx;
-            visit(statectx);
-
-            sourcesmd.methodEscape = statesmd.methodEscape;
-            sourcesmd.allLast = statesmd.allLast;
-        }
-
-        decrementScope();
+        statement.processSource(ctx);
 
         return null;
     }
 
     @Override
     public Void visitIf(final IfContext ctx) {
-        final Metadata.StatementMetadata ifsmd = metadata.getStatementMetadata(ctx);
-
-        final ExpressionContext exprctx = metadata.updateExpressionTree(ctx.expression());
-        final Metadata.ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-        expremd.to = definition.booleanType;
-        visit(exprctx);
-        markCast(expremd);
-
-        if (expremd.postConst != null) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "If statement is not necessary.");
-        }
-
-        final BlockContext blockctx0 = ctx.block(0);
-        final Metadata.StatementMetadata blocksmd0 = metadata.createStatementMetadata(blockctx0);
-        blocksmd0.lastSource = ifsmd.lastSource;
-        blocksmd0.inLoop = ifsmd.inLoop;
-        blocksmd0.lastLoop = ifsmd.lastLoop;
-        incrementScope();
-        visit(blockctx0);
-        decrementScope();
-
-        ifsmd.anyContinue = blocksmd0.anyContinue;
-        ifsmd.anyBreak = blocksmd0.anyBreak;
-
-        ifsmd.count = blocksmd0.count;
-
-        if (ctx.ELSE() != null) {
-            final BlockContext blockctx1 = ctx.block(1);
-            final Metadata.StatementMetadata blocksmd1 = metadata.createStatementMetadata(blockctx1);
-            blocksmd1.lastSource = ifsmd.lastSource;
-            incrementScope();
-            visit(blockctx1);
-            decrementScope();
-
-            ifsmd.methodEscape = blocksmd0.methodEscape && blocksmd1.methodEscape;
-            ifsmd.loopEscape = blocksmd0.loopEscape && blocksmd1.loopEscape;
-            ifsmd.allLast = blocksmd0.allLast && blocksmd1.allLast;
-            ifsmd.anyContinue |= blocksmd1.anyContinue;
-            ifsmd.anyBreak |= blocksmd1.anyBreak;
-
-            ifsmd.count = Math.max(ifsmd.count, blocksmd1.count);
-        }
+        statement.processIf(ctx);
 
         return null;
     }
 
     @Override
     public Void visitWhile(final WhileContext ctx) {
-        final Metadata.StatementMetadata whilesmd = metadata.getStatementMetadata(ctx);
-
-        incrementScope();
-
-        final ExpressionContext exprctx = metadata.updateExpressionTree(ctx.expression());
-        final Metadata.ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-        expremd.to = definition.booleanType;
-        visit(exprctx);
-        markCast(expremd);
-
-        boolean continuous = false;
-
-        if (expremd.postConst != null) {
-            continuous = (boolean)expremd.postConst;
-
-            if (!continuous) {
-                throw new IllegalArgumentException(Metadata.error(ctx) + "The loop will never be executed.");
-            }
-
-            if (ctx.empty() != null) {
-                throw new IllegalArgumentException(Metadata.error(ctx) + "The loop will never exit.");
-            }
-        }
-
-        final BlockContext blockctx = ctx.block();
-
-        if (blockctx != null) {
-            final Metadata.StatementMetadata blocksmd = metadata.createStatementMetadata(blockctx);
-            blocksmd.beginLoop = true;
-            blocksmd.inLoop = true;
-            visit(blockctx);
-
-            if (blocksmd.loopEscape && !blocksmd.anyContinue) {
-                throw new IllegalArgumentException(Metadata.error(ctx) + "All paths escape so the loop is not necessary.");
-            }
-
-            if (continuous && !blocksmd.anyBreak) {
-                whilesmd.methodEscape = true;
-                whilesmd.allLast = true;
-            }
-        }
-
-        whilesmd.count = 1;
-
-        decrementScope();
+        statement.processWhile(ctx);
 
         return null;
     }
 
     @Override
     public Void visitDo(final DoContext ctx) {
-        final Metadata.StatementMetadata dosmd = metadata.getStatementMetadata(ctx);
-
-        incrementScope();
-
-        final BlockContext blockctx = ctx.block();
-        final Metadata.StatementMetadata blocksmd = metadata.createStatementMetadata(blockctx);
-        blocksmd.beginLoop = true;
-        blocksmd.inLoop = true;
-        visit(blockctx);
-
-        if (blocksmd.loopEscape && !blocksmd.anyContinue) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "All paths escape so the loop is not necessary.");
-        }
-
-        final ExpressionContext exprctx = metadata.updateExpressionTree(ctx.expression());
-        final Metadata.ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-        expremd.to = definition.booleanType;
-        visit(exprctx);
-        markCast(expremd);
-
-        if (expremd.postConst != null) {
-            final boolean continuous = (boolean)expremd.postConst;
-
-            if (!continuous) {
-                throw new IllegalArgumentException(Metadata.error(ctx) + "All paths escape so the loop is not necessary.");
-            }
-
-            if (!blocksmd.anyBreak) {
-                dosmd.methodEscape = true;
-                dosmd.allLast = true;
-            }
-        }
-
-        dosmd.count = 1;
-
-        decrementScope();
+        statement.processDo(ctx);
 
         return null;
     }
 
     @Override
     public Void visitFor(final ForContext ctx) {
-        final Metadata.StatementMetadata forsmd = metadata.getStatementMetadata(ctx);
-        boolean continuous = false;
-
-        incrementScope();
-
-        final InitializerContext initctx = ctx.initializer();
-
-        if (initctx != null) {
-            metadata.createStatementMetadata(initctx);
-            visit(initctx);
-        }
-
-        final ExpressionContext exprctx = metadata.updateExpressionTree(ctx.expression());
-
-        if (exprctx != null) {
-            final Metadata.ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-            expremd.to = definition.booleanType;
-            visit(exprctx);
-            markCast(expremd);
-
-            if (expremd.postConst != null) {
-                continuous = (boolean)expremd.postConst;
-
-                if (!continuous) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) + "The loop will never be executed.");
-                }
-
-                if (ctx.empty() != null) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) + "The loop is continuous.");
-                }
-            }
-        } else {
-            continuous = true;
-        }
-
-        final AfterthoughtContext atctx = ctx.afterthought();
-
-        if (atctx != null) {
-            metadata.createStatementMetadata(atctx);
-            visit(atctx);
-        }
-
-        final BlockContext blockctx = ctx.block();
-
-        if (blockctx != null) {
-            final Metadata.StatementMetadata blocksmd = metadata.createStatementMetadata(blockctx);
-            blocksmd.beginLoop = true;
-            blocksmd.inLoop = true;
-            visit(blockctx);
-
-            if (blocksmd.loopEscape && !blocksmd.anyContinue) {
-                throw new IllegalArgumentException(Metadata.error(ctx) + "All paths escape so the loop is not necessary.");
-            }
-
-            if (continuous && !blocksmd.anyBreak) {
-                forsmd.methodEscape = true;
-                forsmd.allLast = true;
-            }
-        }
-
-        forsmd.count = 1;
-
-        decrementScope();
+        statement.processFor(ctx);
 
         return null;
     }
 
     @Override
     public Void visitDecl(final DeclContext ctx) {
-        final Metadata.StatementMetadata declsmd = metadata.getStatementMetadata(ctx);
-
-        final DeclarationContext declctx = ctx.declaration();
-        metadata.createStatementMetadata(declctx);
-        visit(declctx);
-
-        declsmd.count = 1;
+        statement.processDecl(ctx);
 
         return null;
     }
 
     @Override
     public Void visitContinue(final ContinueContext ctx) {
-        final Metadata.StatementMetadata continuesmd = metadata.getStatementMetadata(ctx);
-
-        if (!continuesmd.inLoop) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "Cannot have a continue statement outside of a loop.");
-        }
-
-        if (continuesmd.lastLoop) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "Unnessary continue statement at the end of a loop.");
-        }
-
-        continuesmd.allLast = true;
-        continuesmd.anyContinue = true;
-
-        continuesmd.count = 1;
+        statement.processContinue(ctx);
 
         return null;
     }
 
     @Override
     public Void visitBreak(final BreakContext ctx) {
-        final Metadata.StatementMetadata breaksmd = metadata.getStatementMetadata(ctx);
-
-        if (!breaksmd.inLoop) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "Cannot have a break statement outside of a loop.");
-        }
-
-        breaksmd.loopEscape = true;
-        breaksmd.allLast = true;
-        breaksmd.anyBreak = true;
-
-        breaksmd.count = 1;
+        statement.processBreak(ctx);
 
         return null;
     }
 
     @Override
     public Void visitReturn(final ReturnContext ctx) {
-        final Metadata.StatementMetadata returnsmd = metadata.getStatementMetadata(ctx);
-
-        final ExpressionContext exprctx = metadata.updateExpressionTree(ctx.expression());
-        final Metadata.ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-        expremd.to = definition.objectType;
-        visit(exprctx);
-        markCast(expremd);
-
-        returnsmd.methodEscape = true;
-        returnsmd.loopEscape = true;
-        returnsmd.allLast = true;
-
-        returnsmd.count = 1;
+        statement.processReturn(ctx);
 
         return null;
     }
 
     @Override
     public Void visitTry(final TryContext ctx) {
-        final Metadata.StatementMetadata trysmd = metadata.getStatementMetadata(ctx);
-
-        final BlockContext blockctx = ctx.block();
-        final Metadata.StatementMetadata blocksmd = metadata.createStatementMetadata(blockctx);
-        blocksmd.lastSource = trysmd.lastSource;
-        blocksmd.inLoop = trysmd.inLoop;
-        blocksmd.lastLoop = trysmd.lastLoop;
-        incrementScope();
-        visit(blockctx);
-        decrementScope();
-
-        trysmd.methodEscape = blocksmd.methodEscape;
-        trysmd.loopEscape = blocksmd.loopEscape;
-        trysmd.allLast = blocksmd.allLast;
-        trysmd.anyContinue = blocksmd.anyContinue;
-        trysmd.anyBreak = blocksmd.anyBreak;
-
-        int trapcount = 0;
-
-        for (final TrapContext trapctx : ctx.trap()) {
-            final Metadata.StatementMetadata trapsmd = metadata.createStatementMetadata(trapctx);
-            trapsmd.lastSource = trysmd.lastSource;
-            trapsmd.inLoop = trysmd.inLoop;
-            trapsmd.lastLoop = trysmd.lastLoop;
-            incrementScope();
-            visit(trapctx);
-            decrementScope();
-
-            trysmd.methodEscape &= trapsmd.methodEscape;
-            trysmd.loopEscape &= trapsmd.loopEscape;
-            trysmd.allLast &= trapsmd.allLast;
-            trysmd.anyContinue |= trapsmd.anyContinue;
-            trysmd.anyBreak |= trapsmd.anyBreak;
-
-            trapcount = Math.max(trapcount, trapsmd.count);
-        }
-
-        trysmd.count = blocksmd.count + trapcount;
+        statement.processTry(ctx);
 
         return null;
     }
 
     @Override
     public Void visitThrow(final ThrowContext ctx) {
-        final Metadata.StatementMetadata throwsmd = metadata.getStatementMetadata(ctx);
-
-        final ExpressionContext exprctx = metadata.updateExpressionTree(ctx.expression());
-        final Metadata.ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-        expremd.to = definition.exceptionType;
-        visit(exprctx);
-        markCast(expremd);
-
-        throwsmd.methodEscape = true;
-        throwsmd.loopEscape = true;
-        throwsmd.allLast = true;
-
-        throwsmd.count = 1;
+        statement.processThrow(ctx);
 
         return null;
     }
 
     @Override
     public Void visitExpr(final ExprContext ctx) {
-        final Metadata.StatementMetadata exprsmd = metadata.getStatementMetadata(ctx);
-        final ExpressionContext exprctx = metadata.updateExpressionTree(ctx.expression());
-        final Metadata.ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-        expremd.read = exprsmd.lastSource;
-        visit(exprctx);
-
-        if (!expremd.statement && !exprsmd.lastSource) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "Not a statement.");
-        }
-
-        final boolean rtn = exprsmd.lastSource && expremd.from.sort != Sort.VOID;
-        exprsmd.methodEscape = rtn;
-        exprsmd.loopEscape = rtn;
-        exprsmd.allLast = rtn;
-        expremd.to = rtn ? definition.objectType : expremd.from;
-        markCast(expremd);
-
-        exprsmd.count = 1;
+        statement.processExpr(ctx);
 
         return null;
     }
 
     @Override
     public Void visitMultiple(final MultipleContext ctx) {
-        final Metadata.StatementMetadata multiplesmd = metadata.getStatementMetadata(ctx);
-        final List<StatementContext> statectxs = ctx.statement();
-        final StatementContext lastctx = statectxs.get(statectxs.size() - 1);
-
-        for (StatementContext statectx : statectxs) {
-            if (multiplesmd.allLast) {
-                throw new IllegalArgumentException(Metadata.error(statectx) +
-                    "Statement will never be executed because all prior paths escape.");
-            }
-
-            final Metadata.StatementMetadata statesmd = metadata.createStatementMetadata(statectx);
-            statesmd.lastSource = multiplesmd.lastSource && statectx == lastctx;
-            statesmd.inLoop = multiplesmd.inLoop;
-            statesmd.lastLoop = (multiplesmd.beginLoop || multiplesmd.lastLoop) && statectx == lastctx;
-            visit(statectx);
-
-            multiplesmd.methodEscape = statesmd.methodEscape;
-            multiplesmd.loopEscape = statesmd.loopEscape;
-            multiplesmd.allLast = statesmd.allLast;
-            multiplesmd.anyContinue |= statesmd.anyContinue;
-            multiplesmd.anyBreak |= statesmd.anyBreak;
-
-            multiplesmd.count += statesmd.count;
-        }
+        statement.processMultiple(ctx);
 
         return null;
     }
 
     @Override
     public Void visitSingle(final SingleContext ctx) {
-        final Metadata.StatementMetadata singlesmd = metadata.getStatementMetadata(ctx);
-
-        final StatementContext statectx = ctx.statement();
-        final Metadata.StatementMetadata statesmd = metadata.createStatementMetadata(statectx);
-        statesmd.lastSource = singlesmd.lastSource;
-        statesmd.inLoop = singlesmd.inLoop;
-        statesmd.lastLoop = singlesmd.beginLoop || singlesmd.lastLoop;
-        visit(statectx);
-
-        singlesmd.methodEscape = statesmd.methodEscape;
-        singlesmd.loopEscape = statesmd.loopEscape;
-        singlesmd.allLast = statesmd.allLast;
-        singlesmd.anyContinue = statesmd.anyContinue;
-        singlesmd.anyBreak = statesmd.anyBreak;
-
-        singlesmd.count = statesmd.count;
+        statement.processSingle(ctx);
 
         return null;
     }
 
     @Override
     public Void visitEmpty(final EmptyContext ctx) {
-        throw new UnsupportedOperationException(Metadata.error(ctx) + "Unexpected parser state.");
+        throw new UnsupportedOperationException(AnalyzerUtility.error(ctx) + "Unexpected state.");
     }
 
     @Override
-    public Void visitInitializer(InitializerContext ctx) {
-        final DeclarationContext declctx = ctx.declaration();
-        final ExpressionContext exprctx = metadata.updateExpressionTree(ctx.expression());
+    public Void visitEmptyscope(final EmptyscopeContext ctx) {
+        throw new UnsupportedOperationException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+    }
 
-        if (declctx != null) {
-            metadata.createStatementMetadata(declctx);
-            visit(declctx);
-        } else if (exprctx != null) {
-            final Metadata.ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-            expremd.read = false;
-            visit(exprctx);
-
-            expremd.to = expremd.from;
-            markCast(expremd);
-
-            if (!expremd.statement) {
-                throw new IllegalArgumentException(Metadata.error(exprctx) +
-                    "The intializer of a for loop must be a statement.");
-            }
-        } else {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-        }
+    @Override
+    public Void visitInitializer(final InitializerContext ctx) {
+        statement.processInitializer(ctx);
 
         return null;
     }
 
     @Override
-    public Void visitAfterthought(AfterthoughtContext ctx) {
-        final ExpressionContext exprctx = metadata.updateExpressionTree(ctx.expression());
-
-        if (exprctx != null) {
-            final Metadata.ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-            expremd.read = false;
-            visit(exprctx);
-
-            expremd.to = expremd.from;
-            markCast(expremd);
-
-            if (!expremd.statement) {
-                throw new IllegalArgumentException(Metadata.error(exprctx) +
-                    "The afterthought of a for loop must be a statement.");
-            }
-        }
+    public Void visitAfterthought(final AfterthoughtContext ctx) {
+        statement.processAfterthought(ctx);
 
         return null;
     }
 
     @Override
     public Void visitDeclaration(final DeclarationContext ctx) {
-        final DecltypeContext decltypectx = ctx.decltype();
-        final Metadata.ExpressionMetadata decltypeemd = metadata.createExpressionMetadata(decltypectx);
-        visit(decltypectx);
-
-        for (final DeclvarContext declvarctx : ctx.declvar()) {
-            final Metadata.ExpressionMetadata declvaremd = metadata.createExpressionMetadata(declvarctx);
-            declvaremd.to = decltypeemd.from;
-            visit(declvarctx);
-        }
+        statement.processDeclaration(ctx);
 
         return null;
     }
 
     @Override
     public Void visitDecltype(final DecltypeContext ctx) {
-        final Metadata.ExpressionMetadata decltypeemd = metadata.getExpressionMetadata(ctx);
-
-        final String name = ctx.getText();
-        decltypeemd.from = definition.getType(name);
+        statement.processDecltype(ctx);
 
         return null;
     }
 
     @Override
     public Void visitDeclvar(final DeclvarContext ctx) {
-        final Metadata.ExpressionMetadata declvaremd = metadata.getExpressionMetadata(ctx);
-
-        final String name = ctx.ID().getText();
-        declvaremd.postConst = addVariable(ctx, name, declvaremd.to).slot;
-
-        final ExpressionContext exprctx = metadata.updateExpressionTree(ctx.expression());
-
-        if (exprctx != null) {
-            final Metadata.ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-            expremd.to = declvaremd.to;
-            visit(exprctx);
-            markCast(expremd);
-        }
+        statement.processDeclvar(ctx);
 
         return null;
     }
 
     @Override
     public Void visitTrap(final TrapContext ctx) {
-        final Metadata.StatementMetadata trapsmd = metadata.getStatementMetadata(ctx);
-
-        final String type = ctx.TYPE().getText();
-        trapsmd.exception = definition.getType(type);
-
-        try {
-            trapsmd.exception.clazz.asSubclass(Exception.class);
-        } catch (final ClassCastException exception) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "Invalid exception type [" + trapsmd.exception.name + "].");
-        }
-
-        final String id = ctx.ID().getText();
-        trapsmd.slot = addVariable(ctx, id, trapsmd.exception).slot;
-
-        final BlockContext blockctx = ctx.block();
-
-        if (blockctx != null) {
-            final Metadata.StatementMetadata blocksmd = metadata.createStatementMetadata(blockctx);
-            blocksmd.lastSource = trapsmd.lastSource;
-            blocksmd.inLoop = trapsmd.inLoop;
-            blocksmd.lastLoop = trapsmd.lastLoop;
-            visit(blockctx);
-
-            trapsmd.methodEscape = blocksmd.methodEscape;
-            trapsmd.loopEscape = blocksmd.loopEscape;
-            trapsmd.allLast = blocksmd.allLast;
-            trapsmd.anyContinue = blocksmd.anyContinue;
-            trapsmd.anyBreak = blocksmd.anyBreak;
-        } else if (ctx.emptyscope() == null) {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-        }
+        statement.processTrap(ctx);
 
         return null;
     }
 
     @Override
     public Void visitPrecedence(final PrecedenceContext ctx) {
-        throw new UnsupportedOperationException(Metadata.error(ctx) + "Unexpected parser state.");
+        throw new UnsupportedOperationException(AnalyzerUtility.error(ctx) + "Unexpected state.");
     }
 
     @Override
     public Void visitNumeric(final NumericContext ctx) {
-        final Metadata.ExpressionMetadata numericemd = metadata.getExpressionMetadata(ctx);
-        final boolean negate = ctx.parent instanceof UnaryContext && ((UnaryContext)ctx.parent).SUB() != null;
-
-        if (ctx.DECIMAL() != null) {
-            final String svalue = (negate ? "-" : "") + ctx.DECIMAL().getText();
-
-            if (svalue.endsWith("f") || svalue.endsWith("F")) {
-                try {
-                    numericemd.from = definition.floatType;
-                    numericemd.preConst = Float.parseFloat(svalue.substring(0, svalue.length() - 1));
-                } catch (NumberFormatException exception) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) + "Invalid float constant [" + svalue + "].");
-                }
-            } else {
-                try {
-                    numericemd.from = definition.doubleType;
-                    numericemd.preConst = Double.parseDouble(svalue);
-                } catch (NumberFormatException exception) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) + "Invalid double constant [" + svalue + "].");
-                }
-            }
-        } else {
-            String svalue = negate ? "-" : "";
-            int radix;
-
-            if (ctx.OCTAL() != null) {
-                svalue += ctx.OCTAL().getText();
-                radix = 8;
-            } else if (ctx.INTEGER() != null) {
-                svalue += ctx.INTEGER().getText();
-                radix = 10;
-            } else if (ctx.HEX() != null) {
-                svalue += ctx.HEX().getText();
-                radix = 16;
-            } else {
-                throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-            }
-
-            if (svalue.endsWith("d") || svalue.endsWith("D")) {
-                try {
-                    numericemd.from = definition.doubleType;
-                    numericemd.preConst = Double.parseDouble(svalue.substring(0, svalue.length() - 1));
-                } catch (NumberFormatException exception) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) + "Invalid float constant [" + svalue + "].");
-                }
-            } else if (svalue.endsWith("f") || svalue.endsWith("F")) {
-                try {
-                    numericemd.from = definition.floatType;
-                    numericemd.preConst = Float.parseFloat(svalue.substring(0, svalue.length() - 1));
-                } catch (NumberFormatException exception) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) + "Invalid float constant [" + svalue + "].");
-                }
-            } else if (svalue.endsWith("l") || svalue.endsWith("L")) {
-                try {
-                    numericemd.from = definition.longType;
-                    numericemd.preConst = Long.parseLong(svalue.substring(0, svalue.length() - 1), radix);
-                } catch (NumberFormatException exception) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) + "Invalid long constant [" + svalue + "].");
-                }
-            } else {
-                try {
-                    final Type type = numericemd.to;
-                    final Sort sort = type == null ? Sort.INT : type.sort;
-                    final int value = Integer.parseInt(svalue, radix);
-
-                    if (sort == Sort.BYTE && value >= Byte.MIN_VALUE && value <= Byte.MAX_VALUE) {
-                        numericemd.from = definition.byteType;
-                        numericemd.preConst = (byte)value;
-                    } else if (sort == Sort.CHAR && value >= Character.MIN_VALUE && value <= Character.MAX_VALUE) {
-                        numericemd.from = definition.charType;
-                        numericemd.preConst = (char)value;
-                    } else if (sort == Sort.SHORT && value >= Short.MIN_VALUE && value <= Short.MAX_VALUE) {
-                        numericemd.from = definition.shortType;
-                        numericemd.preConst = (short)value;
-                    } else {
-                        numericemd.from = definition.intType;
-                        numericemd.preConst = value;
-                    }
-                } catch (NumberFormatException exception) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) + "Invalid int constant [" + svalue + "].");
-                }
-            }
-        }
+        expression.processNumeric(ctx);
 
         return null;
     }
 
     @Override
     public Void visitChar(final CharContext ctx) {
-        final Metadata.ExpressionMetadata charemd = metadata.getExpressionMetadata(ctx);
-
-        if (ctx.CHAR() == null) {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-        }
-
-        charemd.preConst = ctx.CHAR().getText().charAt(0);
-        charemd.from = definition.charType;
+        expression.processChar(ctx);
 
         return null;
     }
 
     @Override
     public Void visitTrue(final TrueContext ctx) {
-        final Metadata.ExpressionMetadata trueemd = metadata.getExpressionMetadata(ctx);
-
-        if (ctx.TRUE() == null) {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-        }
-
-        trueemd.preConst = true;
-        trueemd.from = definition.booleanType;
+        expression.processTrue(ctx);
 
         return null;
     }
 
     @Override
     public Void visitFalse(final FalseContext ctx) {
-        final Metadata.ExpressionMetadata falseemd = metadata.getExpressionMetadata(ctx);
-
-        if (ctx.FALSE() == null) {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-        }
-
-        falseemd.preConst = false;
-        falseemd.from = definition.booleanType;
+        expression.processFalse(ctx);
 
         return null;
     }
 
     @Override
     public Void visitNull(final NullContext ctx) {
-        final Metadata.ExpressionMetadata nullemd = metadata.getExpressionMetadata(ctx);
-
-        if (ctx.NULL() == null) {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-        }
-
-        nullemd.isNull = true;
-
-        if (nullemd.to != null) {
-            if (nullemd.to.sort.primitive) {
-                throw new IllegalArgumentException("Cannot cast null to a primitive type [" + nullemd.to.name + "].");
-            }
-
-            nullemd.from = nullemd.to;
-        } else {
-            nullemd.from = definition.objectType;
-        }
+        expression.processNull(ctx);
 
         return null;
     }
 
     @Override
     public Void visitExternal(final ExternalContext ctx) {
-        final Metadata.ExpressionMetadata extemd = metadata.getExpressionMetadata(ctx);
-
-        final ExtstartContext extstartctx = ctx.extstart();
-        final Metadata.ExternalMetadata extstartemd = metadata.createExternalMetadata(extstartctx);
-        extstartemd.read = extemd.read;
-        visit(extstartctx);
-
-        extemd.statement = extstartemd.statement;
-        extemd.preConst = extstartemd.constant;
-        extemd.from = extstartemd.current;
-        extemd.typesafe = extstartemd.current.sort != Sort.DEF;
+        expression.processExternal(ctx);
 
         return null;
     }
 
     @Override
     public Void visitPostinc(final PostincContext ctx) {
-        final Metadata.ExpressionMetadata postincemd = metadata.getExpressionMetadata(ctx);
-
-        final ExtstartContext extstartctx = ctx.extstart();
-        final Metadata.ExternalMetadata extstartemd = metadata.createExternalMetadata(extstartctx);
-        extstartemd.read = postincemd.read;
-        extstartemd.storeExpr = ctx.increment();
-        extstartemd.token = ADD;
-        extstartemd.post = true;
-        visit(extstartctx);
-
-        postincemd.statement = true;
-        postincemd.from = extstartemd.read ? extstartemd.current : definition.voidType;
-        postincemd.typesafe = extstartemd.current.sort != Sort.DEF;
+        expression.processPostinc(ctx);
 
         return null;
     }
 
     @Override
     public Void visitPreinc(final PreincContext ctx) {
-        final Metadata.ExpressionMetadata preincemd = metadata.getExpressionMetadata(ctx);
-
-        final ExtstartContext extstartctx = ctx.extstart();
-        final Metadata.ExternalMetadata extstartemd = metadata.createExternalMetadata(extstartctx);
-        extstartemd.read = preincemd.read;
-        extstartemd.storeExpr = ctx.increment();
-        extstartemd.token = ADD;
-        extstartemd.pre = true;
-        visit(extstartctx);
-
-        preincemd.statement = true;
-        preincemd.from = extstartemd.read ? extstartemd.current : definition.voidType;
-        preincemd.typesafe = extstartemd.current.sort != Sort.DEF;
+        expression.processPreinc(ctx);
 
         return null;
     }
 
     @Override
     public Void visitUnary(final UnaryContext ctx) {
-        final Metadata.ExpressionMetadata unaryemd = metadata.getExpressionMetadata(ctx);
-
-        final ExpressionContext exprctx = metadata.updateExpressionTree(ctx.expression());
-        final Metadata.ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-
-        if (ctx.BOOLNOT() != null) {
-            expremd.to = definition.booleanType;
-            visit(exprctx);
-            markCast(expremd);
-
-            if (expremd.postConst != null) {
-                unaryemd.preConst = !(boolean)expremd.postConst;
-            }
-
-            unaryemd.from = definition.booleanType;
-        } else if (ctx.BWNOT() != null || ctx.ADD() != null || ctx.SUB() != null) {
-            visit(exprctx);
-
-            final Type promote = promoteNumeric(expremd.from, ctx.BWNOT() == null, true);
-
-            if (promote == null) {
-                throw new ClassCastException("Cannot apply [" + ctx.getChild(0).getText() + "] " +
-                    "operation to type [" + expremd.from.name + "].");
-            }
-
-            expremd.to = promote;
-            markCast(expremd);
-
-            if (expremd.postConst != null) {
-                final Sort sort = promote.sort;
-
-                if (ctx.BWNOT() != null) {
-                    if (sort == Sort.INT) {
-                        unaryemd.preConst = ~(int)expremd.postConst;
-                    } else if (sort == Sort.LONG) {
-                        unaryemd.preConst = ~(long)expremd.postConst;
-                    } else {
-                        throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-                    }
-                } else if (ctx.SUB() != null) {
-                    if (exprctx instanceof NumericContext) {
-                        unaryemd.preConst = expremd.postConst;
-                    } else {
-                        if (sort == Sort.INT) {
-                            if (settings.getNumericOverflow()) {
-                                unaryemd.preConst = -(int)expremd.postConst;
-                            } else {
-                                unaryemd.preConst = Math.negateExact((int)expremd.postConst);
-                            }
-                        } else if (sort == Sort.LONG) {
-                            if (settings.getNumericOverflow()) {
-                                unaryemd.preConst = -(long)expremd.postConst;
-                            } else {
-                                unaryemd.preConst = Math.negateExact((long)expremd.postConst);
-                            }
-                        } else if (sort == Sort.FLOAT) {
-                            unaryemd.preConst = -(float)expremd.postConst;
-                        } else if (sort == Sort.DOUBLE) {
-                            unaryemd.preConst = -(double)expremd.postConst;
-                        } else {
-                            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-                        }
-                    }
-                } else if (ctx.ADD() != null) {
-                    if (sort == Sort.INT) {
-                        unaryemd.preConst = +(int)expremd.postConst;
-                    } else if (sort == Sort.LONG) {
-                        unaryemd.preConst = +(long)expremd.postConst;
-                    } else if (sort == Sort.FLOAT) {
-                        unaryemd.preConst = +(float)expremd.postConst;
-                    } else if (sort == Sort.DOUBLE) {
-                        unaryemd.preConst = +(double)expremd.postConst;
-                    } else {
-                        throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-                    }
-                } else {
-                    throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-                }
-            }
-
-            unaryemd.from = promote;
-            unaryemd.typesafe = expremd.typesafe;
-        } else {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-        }
+        expression.processUnary(ctx);
 
         return null;
     }
 
     @Override
     public Void visitCast(final CastContext ctx) {
-        final Metadata.ExpressionMetadata castemd = metadata.getExpressionMetadata(ctx);
-
-        final DecltypeContext decltypectx = ctx.decltype();
-        final Metadata.ExpressionMetadata decltypemd = metadata.createExpressionMetadata(decltypectx);
-        visit(decltypectx);
-
-        final Type type = decltypemd.from;
-        castemd.from = type;
-
-        final ExpressionContext exprctx = metadata.updateExpressionTree(ctx.expression());
-        final Metadata.ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-        expremd.to = type;
-        expremd.explicit = true;
-        visit(exprctx);
-        markCast(expremd);
-
-        if (expremd.postConst != null) {
-            castemd.preConst = expremd.postConst;
-        }
-
-        castemd.typesafe = expremd.typesafe && castemd.from.sort != Sort.DEF;
+        expression.processCast(ctx);
 
         return null;
     }
 
     @Override
     public Void visitBinary(final BinaryContext ctx) {
-        final Metadata.ExpressionMetadata binaryemd = metadata.getExpressionMetadata(ctx);
-
-        final ExpressionContext exprctx0 = metadata.updateExpressionTree(ctx.expression(0));
-        final Metadata.ExpressionMetadata expremd0 = metadata.createExpressionMetadata(exprctx0);
-        visit(exprctx0);
-
-        final ExpressionContext exprctx1 = metadata.updateExpressionTree(ctx.expression(1));
-        final Metadata.ExpressionMetadata expremd1 = metadata.createExpressionMetadata(exprctx1);
-        visit(exprctx1);
-
-        final boolean decimal = ctx.MUL() != null || ctx.DIV() != null || ctx.REM() != null || ctx.SUB() != null;
-        final boolean add = ctx.ADD() != null;
-        final boolean xor = ctx.BWXOR() != null;
-        final Type promote = add ? promoteAdd(expremd0.from, expremd1.from) :
-            xor ? promoteXor(expremd0.from, expremd1.from) :
-                promoteNumeric(expremd0.from, expremd1.from, decimal, true);
-
-        if (promote == null) {
-            throw new ClassCastException("Cannot apply [" + ctx.getChild(1).getText() + "] " +
-                "operation to types [" + expremd0.from.name + "] and [" + expremd1.from.name + "].");
-        }
-
-        final Sort sort = promote.sort;
-        expremd0.to = add && sort == Sort.STRING ? expremd0.from : promote;
-        expremd1.to = add && sort == Sort.STRING ? expremd1.from : promote;
-        markCast(expremd0);
-        markCast(expremd1);
-
-        if (expremd0.postConst != null && expremd1.postConst != null) {
-            if (ctx.MUL() != null) {
-                if (sort == Sort.INT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (int)expremd0.postConst * (int)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Math.multiplyExact((int)expremd0.postConst, (int)expremd1.postConst);
-                    }
-                } else if (sort == Sort.LONG) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (long)expremd0.postConst * (long)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Math.multiplyExact((long)expremd0.postConst, (long)expremd1.postConst);
-                    }
-                } else if (sort == Sort.FLOAT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (float)expremd0.postConst * (float)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.multiplyWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst);
-                    }
-                } else if (sort == Sort.DOUBLE) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (double)expremd0.postConst * (double)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.multiplyWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst);
-                    }
-                } else {
-                    throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-                }
-            } else if (ctx.DIV() != null) {
-                if (sort == Sort.INT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (int)expremd0.postConst / (int)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.divideWithoutOverflow((int)expremd0.postConst, (int)expremd1.postConst);
-                    }
-                } else if (sort == Sort.LONG) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (long)expremd0.postConst / (long)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.divideWithoutOverflow((long)expremd0.postConst, (long)expremd1.postConst);
-                    }
-                } else if (sort == Sort.FLOAT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (float)expremd0.postConst / (float)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.divideWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst);
-                    }
-                } else if (sort == Sort.DOUBLE) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (double)expremd0.postConst / (double)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.divideWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst);
-                    }
-                } else {
-                    throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-                }
-            } else if (ctx.REM() != null) {
-                if (sort == Sort.INT) {
-                    binaryemd.preConst = (int)expremd0.postConst % (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    binaryemd.preConst = (long)expremd0.postConst % (long)expremd1.postConst;
-                } else if (sort == Sort.FLOAT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (float)expremd0.postConst % (float)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.remainderWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst);
-                    }
-                } else if (sort == Sort.DOUBLE) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (double)expremd0.postConst % (double)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.remainderWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst);
-                    }
-                } else {
-                    throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-                }
-            } else if (ctx.ADD() != null) {
-                if (sort == Sort.INT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (int)expremd0.postConst + (int)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Math.addExact((int)expremd0.postConst, (int)expremd1.postConst);
-                    }
-                } else if (sort == Sort.LONG) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (long)expremd0.postConst + (long)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Math.addExact((long)expremd0.postConst, (long)expremd1.postConst);
-                    }
-                } else if (sort == Sort.FLOAT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (float)expremd0.postConst + (float)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.addWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst);
-                    }
-                } else if (sort == Sort.DOUBLE) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (double)expremd0.postConst + (double)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.addWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst);
-                    }
-                } else if (sort == Sort.STRING) {
-                    binaryemd.preConst = "" + expremd0.postConst + expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-                }
-            } else if (ctx.SUB() != null) {
-                if (sort == Sort.INT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (int)expremd0.postConst - (int)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Math.subtractExact((int)expremd0.postConst, (int)expremd1.postConst);
-                    }
-                } else if (sort == Sort.LONG) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (long)expremd0.postConst - (long)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Math.subtractExact((long)expremd0.postConst, (long)expremd1.postConst);
-                    }
-                } else if (sort == Sort.FLOAT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (float)expremd0.postConst - (float)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.subtractWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst);
-                    }
-                } else if (sort == Sort.DOUBLE) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (double)expremd0.postConst - (double)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.subtractWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst);
-                    }
-                } else {
-                    throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-                }
-            } else if (ctx.LSH() != null) {
-                if (sort == Sort.INT) {
-                    binaryemd.preConst = (int)expremd0.postConst << (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    binaryemd.preConst = (long)expremd0.postConst << (long)expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-                }
-            } else if (ctx.RSH() != null) {
-                if (sort == Sort.INT) {
-                    binaryemd.preConst = (int)expremd0.postConst >> (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    binaryemd.preConst = (long)expremd0.postConst >> (long)expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-                }
-            } else if (ctx.USH() != null) {
-                if (sort == Sort.INT) {
-                    binaryemd.preConst = (int)expremd0.postConst >>> (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    binaryemd.preConst = (long)expremd0.postConst >>> (long)expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-                }
-            } else if (ctx.BWAND() != null) {
-                if (sort == Sort.INT) {
-                    binaryemd.preConst = (int)expremd0.postConst & (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    binaryemd.preConst = (long)expremd0.postConst & (long)expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-                }
-            } else if (ctx.BWXOR() != null) {
-                if (sort == Sort.BOOL) {
-                    binaryemd.preConst = (boolean)expremd0.postConst ^ (boolean)expremd1.postConst;
-                } else if (sort == Sort.INT) {
-                    binaryemd.preConst = (int)expremd0.postConst ^ (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    binaryemd.preConst = (long)expremd0.postConst ^ (long)expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-                }
-            } else if (ctx.BWOR() != null) {
-                if (sort == Sort.INT) {
-                    binaryemd.preConst = (int)expremd0.postConst | (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    binaryemd.preConst = (long)expremd0.postConst | (long)expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-                }
-            } else {
-                throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-            }
-        }
-
-        binaryemd.from = promote;
-        binaryemd.typesafe = expremd0.typesafe && expremd1.typesafe;
+        expression.processBinary(ctx);
 
         return null;
     }
 
     @Override
     public Void visitComp(final CompContext ctx) {
-        final Metadata.ExpressionMetadata compemd = metadata.getExpressionMetadata(ctx);
-        final boolean equality = ctx.EQ() != null || ctx.NE() != null;
-        final boolean reference = ctx.EQR() != null || ctx.NER() != null;
-
-        final ExpressionContext exprctx0 = metadata.updateExpressionTree(ctx.expression(0));
-        final Metadata.ExpressionMetadata expremd0 = metadata.createExpressionMetadata(exprctx0);
-        visit(exprctx0);
-
-        final ExpressionContext exprctx1 = metadata.updateExpressionTree(ctx.expression(1));
-        final Metadata.ExpressionMetadata expremd1 = metadata.createExpressionMetadata(exprctx1);
-        visit(exprctx1);
-
-        if (expremd0.isNull && expremd1.isNull) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "Unnecessary comparison of null constants.");
-        }
-
-        final Type promote = equality ? promoteEquality(expremd0.from, expremd1.from) :
-            reference ? promoteReference(expremd0.from, expremd1.from) :
-                promoteNumeric(expremd0.from, expremd1.from, true, true);
-
-        if (promote == null) {
-            throw new ClassCastException("Cannot apply [" + ctx.getChild(1).getText() + "] " +
-                "operation to types [" + expremd0.from.name + "] and [" + expremd1.from.name + "].");
-        }
-
-        expremd0.to = promote;
-        expremd1.to = promote;
-        markCast(expremd0);
-        markCast(expremd1);
-
-        if (expremd0.postConst != null && expremd1.postConst != null) {
-            final Sort sort = promote.sort;
-
-            if (ctx.EQ() != null || ctx.EQR() != null) {
-                if (sort == Sort.BOOL) {
-                    compemd.preConst = (boolean)expremd0.postConst == (boolean)expremd1.postConst;
-                } else if (sort == Sort.INT) {
-                    compemd.preConst = (int)expremd0.postConst == (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    compemd.preConst = (long)expremd0.postConst == (long)expremd1.postConst;
-                } else if (sort == Sort.FLOAT) {
-                    compemd.preConst = (float)expremd0.postConst == (float)expremd1.postConst;
-                } else if (sort == Sort.DOUBLE) {
-                    compemd.preConst = (double)expremd0.postConst == (double)expremd1.postConst;
-                } else {
-                    if (ctx.EQ() != null && !expremd0.isNull && !expremd1.isNull) {
-                        compemd.preConst = expremd0.postConst.equals(expremd1.postConst);
-                    } else if (ctx.EQR() != null) {
-                        compemd.preConst = expremd0.postConst == expremd1.postConst;
-                    }
-                }
-            } else if (ctx.NE() != null || ctx.NER() != null) {
-                if (sort == Sort.BOOL) {
-                    compemd.preConst = (boolean)expremd0.postConst != (boolean)expremd1.postConst;
-                } else if (sort == Sort.INT) {
-                    compemd.preConst = (int)expremd0.postConst != (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    compemd.preConst = (long)expremd0.postConst != (long)expremd1.postConst;
-                } else if (sort == Sort.FLOAT) {
-                    compemd.preConst = (float)expremd0.postConst != (float)expremd1.postConst;
-                } else if (sort == Sort.DOUBLE) {
-                    compemd.preConst = (double)expremd0.postConst != (double)expremd1.postConst;
-                } else {
-                    if (ctx.NE() != null && !expremd0.isNull && !expremd1.isNull) {
-                        compemd.preConst = expremd0.postConst.equals(expremd1.postConst);
-                    } else if (ctx.NER() != null) {
-                        compemd.preConst = expremd0.postConst == expremd1.postConst;
-                    }
-                }
-            } else if (ctx.GTE() != null) {
-                if (sort == Sort.INT) {
-                    compemd.preConst = (int)expremd0.postConst >= (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    compemd.preConst = (long)expremd0.postConst >= (long)expremd1.postConst;
-                } else if (sort == Sort.FLOAT) {
-                    compemd.preConst = (float)expremd0.postConst >= (float)expremd1.postConst;
-                } else if (sort == Sort.DOUBLE) {
-                    compemd.preConst = (double)expremd0.postConst >= (double)expremd1.postConst;
-                }
-            } else if (ctx.GT() != null) {
-                if (sort == Sort.INT) {
-                    compemd.preConst = (int)expremd0.postConst > (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    compemd.preConst = (long)expremd0.postConst > (long)expremd1.postConst;
-                } else if (sort == Sort.FLOAT) {
-                    compemd.preConst = (float)expremd0.postConst > (float)expremd1.postConst;
-                } else if (sort == Sort.DOUBLE) {
-                    compemd.preConst = (double)expremd0.postConst > (double)expremd1.postConst;
-                }
-            } else if (ctx.LTE() != null) {
-                if (sort == Sort.INT) {
-                    compemd.preConst = (int)expremd0.postConst <= (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    compemd.preConst = (long)expremd0.postConst <= (long)expremd1.postConst;
-                } else if (sort == Sort.FLOAT) {
-                    compemd.preConst = (float)expremd0.postConst <= (float)expremd1.postConst;
-                } else if (sort == Sort.DOUBLE) {
-                    compemd.preConst = (double)expremd0.postConst <= (double)expremd1.postConst;
-                }
-            } else if (ctx.LT() != null) {
-                if (sort == Sort.INT) {
-                    compemd.preConst = (int)expremd0.postConst < (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    compemd.preConst = (long)expremd0.postConst < (long)expremd1.postConst;
-                } else if (sort == Sort.FLOAT) {
-                    compemd.preConst = (float)expremd0.postConst < (float)expremd1.postConst;
-                } else if (sort == Sort.DOUBLE) {
-                    compemd.preConst = (double)expremd0.postConst < (double)expremd1.postConst;
-                }
-            } else {
-                throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-            }
-        }
-
-        compemd.from = definition.booleanType;
-        compemd.typesafe = expremd0.typesafe && expremd1.typesafe;
+        expression.processComp(ctx);
 
         return null;
     }
 
     @Override
     public Void visitBool(final BoolContext ctx) {
-        final Metadata.ExpressionMetadata boolemd = metadata.getExpressionMetadata(ctx);
-
-        final ExpressionContext exprctx0 = metadata.updateExpressionTree(ctx.expression(0));
-        final Metadata.ExpressionMetadata expremd0 = metadata.createExpressionMetadata(exprctx0);
-        expremd0.to = definition.booleanType;
-        visit(exprctx0);
-        markCast(expremd0);
-
-        final ExpressionContext exprctx1 = metadata.updateExpressionTree(ctx.expression(1));
-        final Metadata.ExpressionMetadata expremd1 = metadata.createExpressionMetadata(exprctx1);
-        expremd1.to = definition.booleanType;
-        visit(exprctx1);
-        markCast(expremd1);
-
-        if (expremd0.postConst != null && expremd1.postConst != null) {
-            if (ctx.BOOLAND() != null) {
-                boolemd.preConst = (boolean)expremd0.postConst && (boolean)expremd1.postConst;
-            } else if (ctx.BOOLOR() != null) {
-                boolemd.preConst = (boolean)expremd0.postConst || (boolean)expremd1.postConst;
-            } else {
-                throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-            }
-        }
-
-        boolemd.from = definition.booleanType;
-        boolemd.typesafe = expremd0.typesafe && expremd1.typesafe;
+        expression.processBool(ctx);
 
         return null;
     }
 
     @Override
     public Void visitConditional(final ConditionalContext ctx) {
-        final Metadata.ExpressionMetadata condemd = metadata.getExpressionMetadata(ctx);
-
-        final ExpressionContext exprctx0 = metadata.updateExpressionTree(ctx.expression(0));
-        final Metadata.ExpressionMetadata expremd0 = metadata.createExpressionMetadata(exprctx0);
-        expremd0.to = definition.booleanType;
-        visit(exprctx0);
-        markCast(expremd0);
-
-        if (expremd0.postConst != null) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "Unnecessary conditional statement.");
-        }
-
-        final ExpressionContext exprctx1 = metadata.updateExpressionTree(ctx.expression(1));
-        final Metadata.ExpressionMetadata expremd1 = metadata.createExpressionMetadata(exprctx1);
-        expremd1.to = condemd.to;
-        expremd1.explicit = condemd.explicit;
-        visit(exprctx1);
-
-        final ExpressionContext exprctx2 = metadata.updateExpressionTree(ctx.expression(2));
-        final Metadata.ExpressionMetadata expremd2 = metadata.createExpressionMetadata(exprctx2);
-        expremd2.to = condemd.to;
-        expremd2.explicit = condemd.explicit;
-        visit(exprctx2);
-
-        if (condemd.to == null) {
-            final Type promote = promoteConditional(expremd1.from, expremd2.from, expremd1.preConst, expremd2.preConst);
-
-            expremd1.to = promote;
-            expremd2.to = promote;
-            condemd.from = promote;
-        } else {
-            condemd.from = condemd.to;
-        }
-
-        markCast(expremd1);
-        markCast(expremd2);
-
-        condemd.typesafe = expremd0.typesafe && expremd1.typesafe;
+        expression.processConditional(ctx);
 
         return null;
     }
 
     @Override
     public Void visitAssignment(final AssignmentContext ctx) {
-        final Metadata.ExpressionMetadata assignemd = metadata.getExpressionMetadata(ctx);
-
-        final ExtstartContext extstartctx = ctx.extstart();
-        final Metadata.ExternalMetadata extstartemd = metadata.createExternalMetadata(extstartctx);
-
-        extstartemd.read = assignemd.read;
-        extstartemd.storeExpr = metadata.updateExpressionTree(ctx.expression());
-
-        if (ctx.AMUL() != null) {
-            extstartemd.token = MUL;
-        } else if (ctx.ADIV() != null) {
-            extstartemd.token = DIV;
-        } else if (ctx.AREM() != null) {
-            extstartemd.token = REM;
-        } else if (ctx.AADD() != null) {
-            extstartemd.token = ADD;
-        } else if (ctx.ASUB() != null) {
-            extstartemd.token = SUB;
-        } else if (ctx.ALSH() != null) {
-            extstartemd.token = LSH;
-        } else if (ctx.AUSH() != null) {
-            extstartemd.token = USH;
-        } else if (ctx.ARSH() != null) {
-            extstartemd.token = RSH;
-        } else if (ctx.AAND() != null) {
-            extstartemd.token = BWAND;
-        } else if (ctx.AXOR() != null) {
-            extstartemd.token = BWXOR;
-        } else if (ctx.AOR() != null) {
-            extstartemd.token = BWOR;
-        }
-
-        visit(extstartctx);
-
-        assignemd.statement = true;
-        assignemd.from = extstartemd.read ? extstartemd.current : definition.voidType;
-        assignemd.typesafe = extstartemd.current.sort != Sort.DEF;
+        expression.processAssignment(ctx);
 
         return null;
     }
 
     @Override
     public Void visitExtstart(final ExtstartContext ctx) {
-        final ExtprecContext precctx = ctx.extprec();
-        final ExtcastContext castctx = ctx.extcast();
-        final ExttypeContext typectx = ctx.exttype();
-        final ExtvarContext varctx = ctx.extvar();
-        final ExtnewContext newctx = ctx.extnew();
-        final ExtstringContext stringctx = ctx.extstring();
-
-        if (precctx != null) {
-            metadata.createExtNodeMetadata(ctx, precctx);
-            visit(precctx);
-        } else if (castctx != null) {
-            metadata.createExtNodeMetadata(ctx, castctx);
-            visit(castctx);
-        } else if (typectx != null) {
-            metadata.createExtNodeMetadata(ctx, typectx);
-            visit(typectx);
-        } else if (varctx != null) {
-            metadata.createExtNodeMetadata(ctx, varctx);
-            visit(varctx);
-        } else if (newctx != null) {
-            metadata.createExtNodeMetadata(ctx, newctx);
-            visit(newctx);
-        } else if (stringctx != null) {
-            metadata.createExtNodeMetadata(ctx, stringctx);
-            visit(stringctx);
-        } else {
-            throw new IllegalStateException();
-        }
+        external.processExtstart(ctx);
 
         return null;
     }
 
     @Override
     public Void visitExtprec(final ExtprecContext ctx) {
-        final Metadata.ExtNodeMetadata precenmd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = precenmd.parent;
-        final Metadata.ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        final ExtprecContext precctx = ctx.extprec();
-        final ExtcastContext castctx = ctx.extcast();
-        final ExttypeContext typectx = ctx.exttype();
-        final ExtvarContext varctx = ctx.extvar();
-        final ExtnewContext newctx = ctx.extnew();
-        final ExtstringContext stringctx = ctx.extstring();
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        if (dotctx != null || bracectx != null) {
-            ++parentemd.scope;
-        }
-
-        if (precctx != null) {
-            metadata.createExtNodeMetadata(parent, precctx);
-            visit(precctx);
-        } else if (castctx != null) {
-            metadata.createExtNodeMetadata(parent, castctx);
-            visit(castctx);
-        } else if (typectx != null) {
-            metadata.createExtNodeMetadata(parent, typectx);
-            visit(typectx);
-        } else if (varctx != null) {
-            metadata.createExtNodeMetadata(parent, varctx);
-            visit(varctx);
-        } else if (newctx != null) {
-            metadata.createExtNodeMetadata(parent, newctx);
-            visit(newctx);
-        } else if (stringctx != null) {
-            metadata.createExtNodeMetadata(ctx, stringctx);
-            visit(stringctx);
-        } else {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-        }
-
-        parentemd.statement = false;
-
-        if (dotctx != null) {
-            --parentemd.scope;
-
-            metadata.createExtNodeMetadata(parent, dotctx);
-            visit(dotctx);
-        } else if (bracectx != null) {
-            --parentemd.scope;
-
-            metadata.createExtNodeMetadata(parent, bracectx);
-            visit(bracectx);
-        }
+        external.processExtprec(ctx);
 
         return null;
     }
 
     @Override
     public Void visitExtcast(final ExtcastContext ctx) {
-        final Metadata.ExtNodeMetadata castenmd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = castenmd.parent;
-        final Metadata.ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        final ExtprecContext precctx = ctx.extprec();
-        final ExtcastContext castctx = ctx.extcast();
-        final ExttypeContext typectx = ctx.exttype();
-        final ExtvarContext varctx = ctx.extvar();
-        final ExtnewContext newctx = ctx.extnew();
-        final ExtstringContext stringctx = ctx.extstring();
-
-        if (precctx != null) {
-            metadata.createExtNodeMetadata(parent, precctx);
-            visit(precctx);
-        } else if (castctx != null) {
-            metadata.createExtNodeMetadata(parent, castctx);
-            visit(castctx);
-        } else if (typectx != null) {
-            metadata.createExtNodeMetadata(parent, typectx);
-            visit(typectx);
-        } else if (varctx != null) {
-            metadata.createExtNodeMetadata(parent, varctx);
-            visit(varctx);
-        } else if (newctx != null) {
-            metadata.createExtNodeMetadata(parent, newctx);
-            visit(newctx);
-        } else if (stringctx != null) {
-            metadata.createExtNodeMetadata(ctx, stringctx);
-            visit(stringctx);
-        } else {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-        }
-
-        final DecltypeContext declctx = ctx.decltype();
-        final Metadata.ExpressionMetadata declemd = metadata.createExpressionMetadata(declctx);
-        visit(declctx);
-
-        castenmd.castTo = getLegalCast(ctx, parentemd.current, declemd.from, true);
-        castenmd.type = declemd.from;
-        parentemd.current = declemd.from;
-        parentemd.statement = false;
+        external.processExtcast(ctx);
 
         return null;
     }
 
     @Override
     public Void visitExtbrace(final ExtbraceContext ctx) {
-        final Metadata.ExtNodeMetadata braceenmd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = braceenmd.parent;
-        final Metadata.ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        final boolean array = parentemd.current.sort == Sort.ARRAY;
-        final boolean def = parentemd.current.sort == Sort.DEF;
-        boolean map = false;
-        boolean list = false;
-
-        try {
-            parentemd.current.clazz.asSubclass(Map.class);
-            map = true;
-        } catch (ClassCastException exception) {
-            // Do nothing.
-        }
-
-        try {
-            parentemd.current.clazz.asSubclass(List.class);
-            list = true;
-        } catch (ClassCastException exception) {
-            // Do nothing.
-        }
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        braceenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null;
-
-        final ExpressionContext exprctx = metadata.updateExpressionTree(ctx.expression());
-        final Metadata.ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-
-        if (array || def) {
-            expremd.to = array ? definition.intType : definition.objectType;
-            visit(exprctx);
-            markCast(expremd);
-
-            braceenmd.target = "#brace";
-            braceenmd.type = def ? definition.defType :
-                definition.getType(parentemd.current.struct, parentemd.current.type.getDimensions() - 1);
-            analyzeLoadStoreExternal(ctx);
-            parentemd.current = braceenmd.type;
-
-            if (dotctx != null) {
-                metadata.createExtNodeMetadata(parent, dotctx);
-                visit(dotctx);
-            } else if (bracectx != null) {
-                metadata.createExtNodeMetadata(parent, bracectx);
-                visit(bracectx);
-            }
-        } else {
-            final boolean store = braceenmd.last && parentemd.storeExpr != null;
-            final boolean get = parentemd.read || parentemd.token > 0 || !braceenmd.last;
-            final boolean set = braceenmd.last && store;
-
-            Method getter;
-            Method setter;
-            Type valuetype;
-            Type settype;
-
-            if (map) {
-                getter = parentemd.current.struct.methods.get("get");
-                setter = parentemd.current.struct.methods.get("put");
-
-                if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1)) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) +
-                        "Illegal map get shortcut for type [" + parentemd.current.name + "].");
-                }
-
-                if (setter != null && setter.arguments.size() != 2) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) +
-                        "Illegal map set shortcut for type [" + parentemd.current.name + "].");
-                }
-
-                if (getter != null && setter != null && (!getter.arguments.get(0).equals(setter.arguments.get(0))
-                    || !getter.rtn.equals(setter.arguments.get(1)))) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) + "Shortcut argument types must match.");
-                }
-
-                valuetype = setter != null ? setter.arguments.get(0) : getter != null ? getter.arguments.get(0) : null;
-                settype = setter == null ? null : setter.arguments.get(1);
-            } else if (list) {
-                getter = parentemd.current.struct.methods.get("get");
-                setter = parentemd.current.struct.methods.get("set");
-
-                if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1 ||
-                    getter.arguments.get(0).sort != Sort.INT)) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) +
-                        "Illegal list get shortcut for type [" + parentemd.current.name + "].");
-                }
-
-                if (setter != null && (setter.arguments.size() != 2 || setter.arguments.get(0).sort != Sort.INT)) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) +
-                        "Illegal list set shortcut for type [" + parentemd.current.name + "].");
-                }
-
-                if (getter != null && setter != null && (!getter.arguments.get(0).equals(setter.arguments.get(0))
-                    || !getter.rtn.equals(setter.arguments.get(1)))) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) + "Shortcut argument types must match.");
-                }
-
-                valuetype = definition.intType;
-                settype = setter == null ? null : setter.arguments.get(1);
-            } else {
-                throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-            }
-
-            if ((get || set) && (!get || getter != null) && (!set || setter != null)) {
-                expremd.to = valuetype;
-                visit(exprctx);
-                markCast(expremd);
-
-                braceenmd.target = new Object[] {getter, setter, true, null};
-                braceenmd.type = get ? getter.rtn : settype;
-                analyzeLoadStoreExternal(ctx);
-                parentemd.current = get ? getter.rtn : setter.rtn;
-            }
-        }
-
-        if (braceenmd.target == null) {
-            throw new IllegalArgumentException(Metadata.error(ctx) +
-                "Attempting to address a non-array type [" + parentemd.current.name + "] as an array.");
-        }
+        external.processExtbrace(ctx);
 
         return null;
     }
 
     @Override
     public Void visitExtdot(final ExtdotContext ctx) {
-        final Metadata.ExtNodeMetadata dotemnd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = dotemnd.parent;
-
-        final ExtcallContext callctx = ctx.extcall();
-        final ExtfieldContext fieldctx = ctx.extfield();
-
-        if (callctx != null) {
-            metadata.createExtNodeMetadata(parent, callctx);
-            visit(callctx);
-        } else if (fieldctx != null) {
-            metadata.createExtNodeMetadata(parent, fieldctx);
-            visit(fieldctx);
-        }
+        external.processExtdot(ctx);
 
         return null;
     }
 
     @Override
     public Void visitExttype(final ExttypeContext ctx) {
-        final Metadata.ExtNodeMetadata typeenmd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = typeenmd.parent;
-        final Metadata.ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        if (parentemd.current != null) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "Unexpected static type.");
-        }
-
-        final String typestr = ctx.TYPE().getText();
-        typeenmd.type = definition.getType(typestr);
-        parentemd.current = typeenmd.type;
-        parentemd.statik = true;
-
-        final ExtdotContext dotctx = ctx.extdot();
-        metadata.createExtNodeMetadata(parent, dotctx);
-        visit(dotctx);
+        external.processExttype(ctx);
 
         return null;
     }
 
     @Override
     public Void visitExtcall(final ExtcallContext ctx) {
-        final Metadata.ExtNodeMetadata callenmd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = callenmd.parent;
-        final Metadata.ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        callenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null;
-
-        final String name = ctx.EXTID().getText();
-
-        if (parentemd.current.sort == Sort.ARRAY) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "Unexpected call [" + name + "] on an array.");
-        } else if (callenmd.last && parentemd.storeExpr != null) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "Cannot assign a value to a call [" + name + "].");
-        }
-
-        final Struct struct = parentemd.current.struct;
-        final List<ExpressionContext> arguments = ctx.arguments().expression();
-        final int size = arguments.size();
-        Type[] types;
-
-        final Method method = parentemd.statik ? struct.functions.get(name) : struct.methods.get(name);
-        final boolean def = parentemd.current.sort == Sort.DEF;
-
-        if (method == null && !def) {
-            throw new IllegalArgumentException(
-                Metadata.error(ctx) + "Unknown call [" + name + "] on type [" + struct.name + "].");
-        } else if (method != null) {
-            types = new Type[method.arguments.size()];
-            method.arguments.toArray(types);
-
-            callenmd.target = method;
-            callenmd.type = method.rtn;
-            parentemd.statement = !parentemd.read && callenmd.last;
-            parentemd.current = method.rtn;
-
-            if (size != types.length) {
-                throw new IllegalArgumentException(Metadata.error(ctx) + "When calling [" + name + "] on type " +
-                    "[" + struct.name + "] expected [" + types.length + "] arguments," +
-                    " but found [" + arguments.size() + "].");
-            }
-        } else {
-            types = new Type[arguments.size()];
-            Arrays.fill(types, definition.defType);
-
-            callenmd.target = name;
-            callenmd.type = definition.defType;
-            parentemd.statement = !parentemd.read && callenmd.last;
-            parentemd.current = callenmd.type;
-        }
-
-        for (int argument = 0; argument < size; ++argument) {
-            final ExpressionContext exprctx = metadata.updateExpressionTree(arguments.get(argument));
-            final Metadata.ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-            expremd.to = types[argument];
-            visit(exprctx);
-            markCast(expremd);
-        }
-
-        parentemd.statik = false;
-
-        if (dotctx != null) {
-            metadata.createExtNodeMetadata(parent, dotctx);
-            visit(dotctx);
-        } else if (bracectx != null) {
-            metadata.createExtNodeMetadata(parent, bracectx);
-            visit(bracectx);
-        }
+        external.processExtcall(ctx);
 
         return null;
     }
 
     @Override
     public Void visitExtvar(final ExtvarContext ctx) {
-        final Metadata.ExtNodeMetadata varenmd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = varenmd.parent;
-        final Metadata.ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        final String name = ctx.ID().getText();
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        if (parentemd.current != null) {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected variable [" + name + "] load.");
-        }
-
-        varenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null;
-
-        final Variable variable = getVariable(name);
-
-        if (variable == null) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "Unknown variable [" + name + "].");
-        }
-
-        varenmd.target = variable.slot;
-        varenmd.type = variable.type;
-        analyzeLoadStoreExternal(ctx);
-        parentemd.current = varenmd.type;
-
-        if (dotctx != null) {
-            metadata.createExtNodeMetadata(parent, dotctx);
-            visit(dotctx);
-        } else if (bracectx != null) {
-            metadata.createExtNodeMetadata(parent, bracectx);
-            visit(bracectx);
-        }
+        external.processExtvar(ctx);
 
         return null;
     }
 
     @Override
     public Void visitExtfield(final ExtfieldContext ctx) {
-        final Metadata.ExtNodeMetadata memberenmd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = memberenmd.parent;
-        final Metadata.ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        if (ctx.EXTID() == null && ctx.EXTINTEGER() == null) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "Unexpected parser state.");
-        }
-
-        final String value = ctx.EXTID() == null ? ctx.EXTINTEGER().getText() : ctx.EXTID().getText();
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        memberenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null;
-        final boolean store = memberenmd.last && parentemd.storeExpr != null;
-
-        if (parentemd.current == null) {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected field [" + value + "] load.");
-        }
-
-        if (parentemd.current.sort == Sort.ARRAY) {
-            if ("length".equals(value)) {
-                if (!parentemd.read) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) + "Must read array field [length].");
-                } else if (store) {
-                    throw new IllegalArgumentException(
-                        Metadata.error(ctx) + "Cannot write to read-only array field [length].");
-                }
-
-                memberenmd.target = "#length";
-                memberenmd.type = definition.intType;
-                parentemd.current = definition.intType;
-            } else {
-                throw new IllegalArgumentException(Metadata.error(ctx) + "Unexpected array field [" + value + "].");
-            }
-        } else if (parentemd.current.sort == Sort.DEF) {
-            memberenmd.target = value;
-            memberenmd.type = definition.defType;
-            analyzeLoadStoreExternal(ctx);
-            parentemd.current = memberenmd.type;
-        } else {
-            final Struct struct = parentemd.current.struct;
-            final Field field = parentemd.statik ? struct.statics.get(value) : struct.members.get(value);
-
-            if (field != null) {
-                if (store && java.lang.reflect.Modifier.isFinal(field.reflect.getModifiers())) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) + "Cannot write to read-only" +
-                        " field [" + value + "] for type [" + struct.name + "].");
-                }
-
-                memberenmd.target = field;
-                memberenmd.type = field.type;
-                analyzeLoadStoreExternal(ctx);
-                parentemd.current = memberenmd.type;
-            } else {
-                final boolean get = parentemd.read || parentemd.token > 0 || !memberenmd.last;
-                final boolean set = memberenmd.last && store;
-
-                Method getter = struct.methods.get("get" + Character.toUpperCase(value.charAt(0)) + value.substring(1));
-                Method setter = struct.methods.get("set" + Character.toUpperCase(value.charAt(0)) + value.substring(1));
-                Object constant = null;
-
-                if (getter != null && (getter.rtn.sort == Sort.VOID || !getter.arguments.isEmpty())) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) +
-                        "Illegal get shortcut on field [" + value + "] for type [" + struct.name + "].");
-                }
-
-                if (setter != null && (setter.rtn.sort != Sort.VOID || setter.arguments.size() != 1)) {
-                    throw new IllegalArgumentException(Metadata.error(ctx) +
-                        "Illegal set shortcut on field [" + value + "] for type [" + struct.name + "].");
-                }
-
-                Type settype = setter == null ? null : setter.arguments.get(0);
-
-                if (getter == null && setter == null) {
-                    if (ctx.EXTID() != null) {
-                        try {
-                            parentemd.current.clazz.asSubclass(Map.class);
-
-                            getter = parentemd.current.struct.methods.get("get");
-                            setter = parentemd.current.struct.methods.get("put");
-
-                            if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1 ||
-                                getter.arguments.get(0).sort != Sort.STRING)) {
-                                throw new IllegalArgumentException(Metadata.error(ctx) +
-                                    "Illegal map get shortcut [" + value + "] for type [" + struct.name + "].");
-                            }
-
-                            if (setter != null && (setter.arguments.size() != 2 ||
-                                setter.arguments.get(0).sort != Sort.STRING)) {
-                                throw new IllegalArgumentException(Metadata.error(ctx) +
-                                    "Illegal map set shortcut [" + value + "] for type [" + struct.name + "].");
-                            }
-
-                            if (getter != null && setter != null && !getter.rtn.equals(setter.arguments.get(1))) {
-                                throw new IllegalArgumentException(Metadata.error(ctx) + "Shortcut argument types must match.");
-                            }
-
-                            settype = setter == null ? null : setter.arguments.get(1);
-                            constant = value;
-                        } catch (ClassCastException exception) {
-                            //Do nothing.
-                        }
-                    } else if (ctx.EXTINTEGER() != null) {
-                        try {
-                            parentemd.current.clazz.asSubclass(List.class);
-
-                            getter = parentemd.current.struct.methods.get("get");
-                            setter = parentemd.current.struct.methods.get("set");
-
-                            if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1 ||
-                                getter.arguments.get(0).sort != Sort.INT)) {
-                                throw new IllegalArgumentException(Metadata.error(ctx) +
-                                    "Illegal list get shortcut [" + value + "] for type [" + struct.name + "].");
-                            }
-
-                            if (setter != null && (setter.rtn.sort != Sort.VOID || setter.arguments.size() != 2 ||
-                                setter.arguments.get(0).sort != Sort.INT)) {
-                                throw new IllegalArgumentException(Metadata.error(ctx) +
-                                    "Illegal list set shortcut [" + value + "] for type [" + struct.name + "].");
-                            }
-
-                            if (getter != null && setter != null && !getter.rtn.equals(setter.arguments.get(1))) {
-                                throw new IllegalArgumentException(Metadata.error(ctx) + "Shortcut argument types must match.");
-                            }
-
-                            settype = setter == null ? null : setter.arguments.get(1);
-
-                            try {
-                                constant = Integer.parseInt(value);
-                            } catch (NumberFormatException exception) {
-                                throw new IllegalArgumentException(Metadata.error(ctx) +
-                                    "Illegal list shortcut value [" + value + "].");
-                            }
-                        } catch (ClassCastException exception) {
-                            //Do nothing.
-                        }
-                    } else {
-                        throw new IllegalStateException(Metadata.error(ctx) + "Unexpected parser state.");
-                    }
-                }
-
-                if ((get || set) && (!get || getter != null) && (!set || setter != null)) {
-                    memberenmd.target = new Object[] {getter, setter, constant != null, constant};
-                    memberenmd.type = get ? getter.rtn : settype;
-                    analyzeLoadStoreExternal(ctx);
-                    parentemd.current = get ? getter.rtn : setter.rtn;
-                }
-            }
-
-            if (memberenmd.target == null) {
-                throw new IllegalArgumentException(
-                    Metadata.error(ctx) + "Unknown field [" + value + "] for type [" + struct.name + "].");
-            }
-        }
-
-        parentemd.statik = false;
-
-        if (dotctx != null) {
-            metadata.createExtNodeMetadata(parent, dotctx);
-            visit(dotctx);
-        } else if (bracectx != null) {
-            metadata.createExtNodeMetadata(parent, bracectx);
-            visit(bracectx);
-        }
+        external.processExtfield(ctx);
 
         return null;
     }
 
     @Override
-    public Void visitExtnew(ExtnewContext ctx) {
-        final Metadata.ExtNodeMetadata newenmd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = newenmd.parent;
-        final Metadata.ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        newenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null;
-
-        final String name = ctx.TYPE().getText();
-        final Struct struct = definition.structs.get(name);
-
-        if (parentemd.current != null) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "Unexpected new call.");
-        } else if (struct == null) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "Specified type [" + name + "] not found.");
-        } else if (newenmd.last && parentemd.storeExpr != null) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "Cannot assign a value to a new call.");
-        }
-
-        final boolean newclass = ctx.arguments() != null;
-        final boolean newarray = !ctx.expression().isEmpty();
-
-        final List<ExpressionContext> arguments = newclass ? ctx.arguments().expression() : ctx.expression();
-        final int size = arguments.size();
-
-        Type[] types;
-
-        if (newarray) {
-            if (!parentemd.read) {
-                throw new IllegalArgumentException(Metadata.error(ctx) + "A newly created array must be assigned.");
-            }
-
-            types = new Type[size];
-            Arrays.fill(types, definition.intType);
-
-            newenmd.target = "#makearray";
-
-            if (size > 1) {
-                newenmd.type = definition.getType(struct, size);
-                parentemd.current = newenmd.type;
-            } else if (size == 1) {
-                newenmd.type = definition.getType(struct, 0);
-                parentemd.current = definition.getType(struct, 1);
-            } else {
-                throw new IllegalArgumentException(Metadata.error(ctx) + "A newly created array cannot have zero dimensions.");
-            }
-        } else if (newclass) {
-            final Constructor constructor = struct.constructors.get("new");
-
-            if (constructor != null) {
-                types = new Type[constructor.arguments.size()];
-                constructor.arguments.toArray(types);
-
-                newenmd.target = constructor;
-                newenmd.type = definition.getType(struct, 0);
-                parentemd.statement = !parentemd.read && newenmd.last;
-                parentemd.current = newenmd.type;
-            } else {
-                throw new IllegalArgumentException(
-                    Metadata.error(ctx) + "Unknown new call on type [" + struct.name + "].");
-            }
-        } else {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "Unknown parser state.");
-        }
-
-        if (size != types.length) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "When calling [" + name + "] on type " +
-                "[" + struct.name + "] expected [" + types.length + "] arguments," +
-                " but found [" + arguments.size() + "].");
-        }
-
-        for (int argument = 0; argument < size; ++argument) {
-            final ExpressionContext exprctx = metadata.updateExpressionTree(arguments.get(argument));
-            final Metadata.ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-            expremd.to = types[argument];
-            visit(exprctx);
-            markCast(expremd);
-        }
-
-        if (dotctx != null) {
-            metadata.createExtNodeMetadata(parent, dotctx);
-            visit(dotctx);
-        } else if (bracectx != null) {
-            metadata.createExtNodeMetadata(parent, bracectx);
-            visit(bracectx);
-        }
+    public Void visitExtnew(final ExtnewContext ctx) {
+        external.processExtnew(ctx);
 
         return null;
     }
 
     @Override
     public Void visitExtstring(final ExtstringContext ctx) {
-        final Metadata.ExtNodeMetadata memberenmd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = memberenmd.parent;
-        final Metadata.ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        final String string = ctx.STRING().getText();
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        memberenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null;
-        final boolean store = memberenmd.last && parentemd.storeExpr != null;
-
-        if (parentemd.current != null) {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected String constant [" + string + "].");
-        }
-
-        if (!parentemd.read) {
-            throw new IllegalArgumentException(Metadata.error(ctx) + "Must read String constant [" + string + "].");
-        } else if (store) {
-            throw new IllegalArgumentException(
-                Metadata.error(ctx) + "Cannot write to read-only String constant [" + string + "].");
-        }
-
-        memberenmd.target = string;
-        memberenmd.type = definition.stringType;
-        parentemd.current = definition.stringType;
-
-        if (memberenmd.last) {
-            parentemd.constant = string;
-        }
-
-        if (dotctx != null) {
-            metadata.createExtNodeMetadata(parent, dotctx);
-            visit(dotctx);
-        } else if (bracectx != null) {
-            metadata.createExtNodeMetadata(parent, bracectx);
-            visit(bracectx);
-        }
+        external.processExtstring(ctx);
 
         return null;
     }
 
     @Override
     public Void visitArguments(final ArgumentsContext ctx) {
-        throw new UnsupportedOperationException(Metadata.error(ctx) + "Unexpected parser state.");
+        throw new UnsupportedOperationException(AnalyzerUtility.error(ctx) + "Unexpected state.");
     }
 
     @Override
-    public Void visitIncrement(IncrementContext ctx) {
-        final Metadata.ExpressionMetadata incremd = metadata.getExpressionMetadata(ctx);
-        final Sort sort = incremd.to == null ? null : incremd.to.sort;
-        final boolean positive = ctx.INCR() != null;
-
-        if (incremd.to == null) {
-            incremd.preConst = positive ? 1 : -1;
-            incremd.from = definition.intType;
-        } else {
-            switch (sort) {
-                case LONG:
-                    incremd.preConst = positive ? 1L : -1L;
-                    incremd.from = definition.longType;
-                    break;
-                case FLOAT:
-                    incremd.preConst = positive ? 1.0F : -1.0F;
-                    incremd.from = definition.floatType;
-                    break;
-                case DOUBLE:
-                    incremd.preConst = positive ? 1.0 : -1.0;
-                    incremd.from = definition.doubleType;
-                    break;
-                default:
-                    incremd.preConst = positive ? 1 : -1;
-                    incremd.from = definition.intType;
-            }
-        }
+    public Void visitIncrement(final IncrementContext ctx) {
+        expression.processIncrement(ctx);
 
         return null;
     }
-
-    private void analyzeLoadStoreExternal(final ParserRuleContext source) {
-        final Metadata.ExtNodeMetadata extenmd = metadata.getExtNodeMetadata(source);
-        final ParserRuleContext parent = extenmd.parent;
-        final Metadata.ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        if (extenmd.last && parentemd.storeExpr != null) {
-            final ParserRuleContext store = parentemd.storeExpr;
-            final Metadata.ExpressionMetadata storeemd = metadata.createExpressionMetadata(parentemd.storeExpr);
-            final int token = parentemd.token;
-
-            if (token > 0) {
-                visit(store);
-
-                final boolean add = token == ADD;
-                final boolean xor = token == BWAND || token == BWXOR || token == BWOR;
-                final boolean decimal = token == MUL || token == DIV || token == REM || token == SUB;
-
-                extenmd.promote = add ? promoteAdd(extenmd.type, storeemd.from) :
-                    xor ? promoteXor(extenmd.type, storeemd.from) :
-                        promoteNumeric(extenmd.type, storeemd.from, decimal, true);
-
-                if (extenmd.promote == null) {
-                    throw new IllegalArgumentException("Cannot apply compound assignment to " +
-                        "types [" + extenmd.type.name + "] and [" + storeemd.from.name + "].");
-                }
-
-                extenmd.castFrom = getLegalCast(source, extenmd.type, extenmd.promote, false);
-                extenmd.castTo = getLegalCast(source, extenmd.promote, extenmd.type, true);
-
-                storeemd.to = add && extenmd.promote.sort == Sort.STRING ? storeemd.from : extenmd.promote;
-                markCast(storeemd);
-            } else {
-                storeemd.to = extenmd.type;
-                visit(store);
-                markCast(storeemd);
-            }
-        }
-    }
-
-    private void markCast(final Metadata.ExpressionMetadata emd) {
-        if (emd.from == null) {
-            throw new IllegalStateException(Metadata.error(emd.source) + "From cast type should never be null.");
-        }
-
-        if (emd.to != null) {
-            emd.cast = getLegalCast(emd.source, emd.from, emd.to, emd.explicit || !emd.typesafe);
-
-            if (emd.preConst != null && emd.to.sort.constant) {
-                emd.postConst = constCast(emd.source, emd.preConst, emd.cast);
-            }
-        } else {
-            throw new IllegalStateException(Metadata.error(emd.source) + "To cast type should never be null.");
-        }
-    }
-
-    private Cast getLegalCast(final ParserRuleContext source, final Type from, final Type to, final boolean explicit) {
-        final Cast cast = new Cast(from, to);
-
-        if (from.equals(to)) {
-            return cast;
-        }
-
-        if (from.sort == Sort.DEF && to.sort != Sort.VOID || from.sort != Sort.VOID && to.sort == Sort.DEF) {
-            final Transform transform = definition.transforms.get(cast);
-
-            if (transform != null) {
-                return transform;
-            }
-
-            return cast;
-        }
-
-        switch (from.sort) {
-            case BOOL:
-                switch (to.sort) {
-                    case OBJECT:
-                    case BOOL_OBJ:
-                        return checkTransform(source, cast);
-                }
-
-                break;
-            case BYTE:
-                switch (to.sort) {
-                    case SHORT:
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                        return cast;
-                    case CHAR:
-                        if (explicit)
-                            return cast;
-
-                        break;
-                    case OBJECT:
-                    case NUMBER:
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
-                    case CHAR_OBJ:
-                        if (explicit)
-                            return checkTransform(source, cast);
-
-                        break;
-                }
-
-                break;
-            case SHORT:
-                switch (to.sort) {
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                        return cast;
-                    case BYTE:
-                    case CHAR:
-                        if (explicit)
-                            return cast;
-
-                        break;
-                    case OBJECT:
-                    case NUMBER:
-                    case SHORT_OBJ:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
-                    case BYTE_OBJ:
-                    case CHAR_OBJ:
-                        if (explicit)
-                            return checkTransform(source, cast);
-
-                        break;
-                }
-
-                break;
-            case CHAR:
-                switch (to.sort) {
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                        return cast;
-                    case BYTE:
-                    case SHORT:
-                        if (explicit)
-                            return cast;
-
-                        break;
-                    case OBJECT:
-                    case NUMBER:
-                    case CHAR_OBJ:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                        if (explicit)
-                            return checkTransform(source, cast);
-
-                        break;
-                }
-
-                break;
-            case INT:
-                switch (to.sort) {
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                        return cast;
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                        if (explicit)
-                            return cast;
-
-                        break;
-                    case OBJECT:
-                    case NUMBER:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case CHAR_OBJ:
-                        if (explicit)
-                            return checkTransform(source, cast);
-
-                        break;
-                }
-
-                break;
-            case LONG:
-                switch (to.sort) {
-                    case FLOAT:
-                    case DOUBLE:
-                        return cast;
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                    case INT:
-                        if (explicit)
-                            return cast;
-
-                        break;
-                    case OBJECT:
-                    case NUMBER:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case CHAR_OBJ:
-                    case INT_OBJ:
-                        if (explicit)
-                            return checkTransform(source, cast);
-
-                        break;
-                }
-
-                break;
-            case FLOAT:
-                switch (to.sort) {
-                    case DOUBLE:
-                        return cast;
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                    case INT:
-                    case LONG:
-                        if (explicit)
-                            return cast;
-
-                        break;
-                    case OBJECT:
-                    case NUMBER:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case CHAR_OBJ:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                        if (explicit)
-                            return checkTransform(source, cast);
-
-                        break;
-                }
-
-                break;
-            case DOUBLE:
-                switch (to.sort) {
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                        if (explicit)
-                            return cast;
-
-                        break;
-                    case OBJECT:
-                    case NUMBER:
-                    case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case CHAR_OBJ:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                        if (explicit)
-                            return checkTransform(source, cast);
-
-                        break;
-                }
-
-                break;
-            case OBJECT:
-            case NUMBER:
-                switch (to.sort) {
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                        if (explicit)
-                            return checkTransform(source, cast);
-
-                        break;
-                }
-
-                break;
-            case BOOL_OBJ:
-                switch (to.sort) {
-                    case BOOL:
-                        return checkTransform(source, cast);
-                }
-
-                break;
-            case BYTE_OBJ:
-                switch (to.sort) {
-                    case BYTE:
-                    case SHORT:
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                    case SHORT_OBJ:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
-                    case CHAR:
-                    case CHAR_OBJ:
-                        if (explicit)
-                            return checkTransform(source, cast);
-
-                        break;
-                }
-
-                break;
-            case SHORT_OBJ:
-                switch (to.sort) {
-                    case SHORT:
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
-                    case BYTE:
-                    case CHAR:
-                    case BYTE_OBJ:
-                    case CHAR_OBJ:
-                        if (explicit)
-                            return checkTransform(source, cast);
-
-                        break;
-                }
-
-                break;
-            case CHAR_OBJ:
-                switch (to.sort) {
-                    case CHAR:
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
-                    case BYTE:
-                    case SHORT:
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                        if (explicit)
-                            return checkTransform(source, cast);
-
-                        break;
-                }
-
-                break;
-            case INT_OBJ:
-                switch (to.sort) {
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case CHAR_OBJ:
-                        if (explicit)
-                            return checkTransform(source, cast);
-
-                        break;
-                }
-
-                break;
-            case LONG_OBJ:
-                switch (to.sort) {
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                    case INT:
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case CHAR_OBJ:
-                    case INT_OBJ:
-                        if (explicit)
-                            return checkTransform(source, cast);
-
-                        break;
-                }
-
-                break;
-            case FLOAT_OBJ:
-                switch (to.sort) {
-                    case FLOAT:
-                    case DOUBLE:
-                    case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                    case INT:
-                    case LONG:
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case CHAR_OBJ:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                        if (explicit)
-                            return checkTransform(source, cast);
-
-                        break;
-                }
-
-                break;
-            case DOUBLE_OBJ:
-                switch (to.sort) {
-                    case DOUBLE:
-                        return checkTransform(source, cast);
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case CHAR_OBJ:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                        if (explicit)
-                            return checkTransform(source, cast);
-
-                        break;
-                }
-
-                break;
-        }
-
-        try {
-            from.clazz.asSubclass(to.clazz);
-
-            return cast;
-        } catch (final ClassCastException cce0) {
-            try {
-                if (explicit) {
-                    to.clazz.asSubclass(from.clazz);
-
-                    return cast;
-                } else {
-                    throw new ClassCastException(
-                        Metadata.error(source) + "Cannot cast from [" + from.name + "] to [" + to.name + "].");
-                }
-            } catch (final ClassCastException cce1) {
-                throw new ClassCastException(
-                    Metadata.error(source) + "Cannot cast from [" + from.name + "] to [" + to.name + "].");
-            }
-        }
-    }
-
-    private Transform checkTransform(final ParserRuleContext source, final Cast cast) {
-        final Transform transform = definition.transforms.get(cast);
-
-        if (transform == null) {
-            throw new ClassCastException(
-                Metadata.error(source) + "Cannot cast from [" + cast.from.name + "] to [" + cast.to.name + "].");
-        }
-
-        return transform;
-    }
-
-    private Object constCast(final ParserRuleContext source, final Object constant, final Cast cast) {
-        if (cast instanceof Transform) {
-            final Transform transform = (Transform)cast;
-            return invokeTransform(source, transform, constant);
-        } else {
-            final Sort fsort = cast.from.sort;
-            final Sort tsort = cast.to.sort;
-
-            if (fsort == tsort) {
-                return constant;
-            } else if (fsort.numeric && tsort.numeric) {
-                Number number;
-
-                if (fsort == Sort.CHAR) {
-                    number = (int)(char)constant;
-                } else {
-                    number = (Number)constant;
-                }
-
-                switch (tsort) {
-                    case BYTE:   return number.byteValue();
-                    case SHORT:  return number.shortValue();
-                    case CHAR:   return (char)number.intValue();
-                    case INT:    return number.intValue();
-                    case LONG:   return number.longValue();
-                    case FLOAT:  return number.floatValue();
-                    case DOUBLE: return number.doubleValue();
-                    default:
-                        throw new IllegalStateException(Metadata.error(source) + "Expected numeric type for cast.");
-                }
-            } else {
-                throw new IllegalStateException(Metadata.error(source) + "No valid constant cast from " +
-                    "[" + cast.from.clazz.getCanonicalName() + "] to " +
-                    "[" + cast.to.clazz.getCanonicalName() + "].");
-            }
-        }
-    }
-
-    private Object invokeTransform(final ParserRuleContext source, final Transform transform, final Object object) {
-        final Method method = transform.method;
-        final java.lang.reflect.Method jmethod = method.reflect;
-        final int modifiers = jmethod.getModifiers();
-
-        try {
-            if (java.lang.reflect.Modifier.isStatic(modifiers)) {
-                return jmethod.invoke(null, object);
-            } else {
-                return jmethod.invoke(object);
-            }
-        } catch (IllegalAccessException | IllegalArgumentException |
-            java.lang.reflect.InvocationTargetException | NullPointerException |
-            ExceptionInInitializerError exception) {
-            throw new IllegalStateException(Metadata.error(source) + "Unable to invoke transform to cast constant from " +
-                "[" + transform.from.name + "] to [" + transform.to.name + "].");
-        }
-    }
-
-    private Type promoteNumeric(final Type from, boolean decimal, boolean primitive) {
-        final Sort sort = from.sort;
-
-        if (sort == Sort.DEF) {
-            return definition.defType;
-        } else if ((sort == Sort.DOUBLE || sort == Sort.DOUBLE_OBJ || sort == Sort.NUMBER) && decimal) {
-            return primitive ? definition.doubleType : definition.doubleobjType;
-        } else if ((sort == Sort.FLOAT || sort == Sort.FLOAT_OBJ) && decimal) {
-            return primitive ? definition.floatType : definition.floatobjType;
-        } else if (sort == Sort.LONG || sort == Sort.LONG_OBJ || sort == Sort.NUMBER) {
-            return primitive ? definition.longType : definition.longobjType;
-        } else if (sort.numeric) {
-            return primitive ? definition.intType : definition.intobjType;
-        }
-
-        return null;
-    }
-
-    private Type promoteNumeric(final Type from0, final Type from1, boolean decimal, boolean primitive) {
-        final Sort sort0 = from0.sort;
-        final Sort sort1 = from1.sort;
-
-        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return definition.defType;
-        }
-
-        if (decimal) {
-            if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ || sort0 == Sort.NUMBER ||
-                sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ || sort1 == Sort.NUMBER) {
-                return primitive ? definition.doubleType : definition.doubleobjType;
-            } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) {
-                return primitive ? definition.floatType : definition.floatobjType;
-            }
-        }
-
-        if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ || sort0 == Sort.NUMBER ||
-            sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ || sort1 == Sort.NUMBER) {
-            return primitive ? definition.longType : definition.longobjType;
-        } else if (sort0.numeric && sort1.numeric) {
-            return primitive ? definition.intType : definition.intobjType;
-        }
-
-        return null;
-    }
-
-    private Type promoteAdd(final Type from0, final Type from1) {
-        final Sort sort0 = from0.sort;
-        final Sort sort1 = from1.sort;
-
-        if (sort0 == Sort.STRING || sort1 == Sort.STRING) {
-            return definition.stringType;
-        }
-
-        return promoteNumeric(from0, from1, true, true);
-    }
-
-    private Type promoteXor(final Type from0, final Type from1) {
-        final Sort sort0 = from0.sort;
-        final Sort sort1 = from1.sort;
-
-        if (sort0.bool || sort1.bool) {
-            return definition.booleanType;
-        }
-
-        return promoteNumeric(from0, from1, false, true);
-    }
-
-    private Type promoteEquality(final Type from0, final Type from1) {
-        final Sort sort0 = from0.sort;
-        final Sort sort1 = from1.sort;
-
-        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return definition.defType;
-        }
-
-        final boolean primitive = sort0.primitive && sort1.primitive;
-
-        if (sort0.bool && sort1.bool) {
-            return primitive ? definition.booleanType : definition.booleanobjType;
-        }
-
-        if (sort0.numeric && sort1.numeric) {
-            return promoteNumeric(from0, from1, true, primitive);
-        }
-
-        return definition.objectType;
-    }
-
-    private Type promoteReference(final Type from0, final Type from1) {
-        final Sort sort0 = from0.sort;
-        final Sort sort1 = from1.sort;
-
-        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return definition.defType;
-        }
-
-        if (sort0.primitive && sort1.primitive) {
-            if (sort0.bool && sort1.bool) {
-                return definition.booleanType;
-            }
-
-            if (sort0.numeric && sort1.numeric) {
-                return promoteNumeric(from0, from1, true, true);
-            }
-        }
-
-        return definition.objectType;
-    }
-
-    private Type promoteConditional(final Type from0, final Type from1, final Object const0, final Object const1) {
-        if (from0.equals(from1)) {
-            return from0;
-        }
-
-        final Sort sort0 = from0.sort;
-        final Sort sort1 = from1.sort;
-
-        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return definition.defType;
-        }
-
-        final boolean primitive = sort0.primitive && sort1.primitive;
-
-        if (sort0.bool && sort1.bool) {
-            return primitive ? definition.booleanType : definition.booleanobjType;
-        }
-
-        if (sort0.numeric && sort1.numeric) {
-            if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ || sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ) {
-                return primitive ? definition.doubleType : definition.doubleobjType;
-            } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) {
-                return primitive ? definition.floatType : definition.floatobjType;
-            } else if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ || sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ) {
-                return sort0.primitive && sort1.primitive ? definition.longType : definition.longobjType;
-            } else {
-                if (sort0 == Sort.BYTE || sort0 == Sort.BYTE_OBJ) {
-                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
-                        return primitive ? definition.byteType : definition.byteobjType;
-                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
-                        if (const1 != null) {
-                            final short constant = (short)const1;
-
-                            if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
-                            }
-                        }
-
-                        return primitive ? definition.shortType : definition.shortobjType;
-                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? definition.intType : definition.intobjType;
-                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
-                        if (const1 != null) {
-                            final int constant = (int)const1;
-
-                            if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
-                            }
-                        }
-
-                        return primitive ? definition.intType : definition.intobjType;
-                    }
-                } else if (sort0 == Sort.SHORT || sort0 == Sort.SHORT_OBJ) {
-                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
-                        if (const0 != null) {
-                            final short constant = (short)const0;
-
-                            if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
-                            }
-                        }
-
-                        return primitive ? definition.shortType : definition.shortobjType;
-                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
-                        return primitive ? definition.shortType : definition.shortobjType;
-                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? definition.intType : definition.intobjType;
-                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
-                        if (const1 != null) {
-                            final int constant = (int)const1;
-
-                            if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
-                                return primitive ? definition.shortType : definition.shortobjType;
-                            }
-                        }
-
-                        return primitive ? definition.intType : definition.intobjType;
-                    }
-                } else if (sort0 == Sort.CHAR || sort0 == Sort.CHAR_OBJ) {
-                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
-                        return primitive ? definition.intType : definition.intobjType;
-                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
-                        return primitive ? definition.intType : definition.intobjType;
-                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? definition.charType : definition.charobjType;
-                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
-                        if (const1 != null) {
-                            final int constant = (int)const1;
-
-                            if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
-                            }
-                        }
-
-                        return primitive ? definition.intType : definition.intobjType;
-                    }
-                } else if (sort0 == Sort.INT || sort0 == Sort.INT_OBJ) {
-                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
-                        if (const0 != null) {
-                            final int constant = (int)const0;
-
-                            if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
-                            }
-                        }
-
-                        return primitive ? definition.intType : definition.intobjType;
-                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
-                        if (const0 != null) {
-                            final int constant = (int)const0;
-
-                            if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
-                            }
-                        }
-
-                        return primitive ? definition.intType : definition.intobjType;
-                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        if (const0 != null) {
-                            final int constant = (int)const0;
-
-                            if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
-                            }
-                        }
-
-                        return primitive ? definition.intType : definition.intobjType;
-                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
-                        return primitive ? definition.intType : definition.intobjType;
-                    }
-                }
-            }
-        }
-
-        final Pair pair = new Pair(from0, from1);
-        final Type bound = definition.bounds.get(pair);
-
-        return bound == null ? definition.objectType : bound;
-    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
new file mode 100644
index 00000000000..46a510bc6bb
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -0,0 +1,563 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.elasticsearch.painless.Definition.Cast;
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Definition.Transform;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Metadata.ExpressionMetadata;
+
+class AnalyzerCaster {
+    private final Definition definition;
+
+    AnalyzerCaster(final Definition definition) {
+        this.definition = definition;
+    }
+
+    void markCast(final ExpressionMetadata emd) {
+        if (emd.from == null) {
+            throw new IllegalStateException(AnalyzerUtility.error(emd.source) + "From cast type should never be null.");
+        }
+
+        if (emd.to != null) {
+            emd.cast = getLegalCast(emd.source, emd.from, emd.to, emd.explicit || !emd.typesafe);
+
+            if (emd.preConst != null && emd.to.sort.constant) {
+                emd.postConst = constCast(emd.source, emd.preConst, emd.cast);
+            }
+        } else {
+            throw new IllegalStateException(AnalyzerUtility.error(emd.source) + "To cast type should never be null.");
+        }
+    }
+
+    Cast getLegalCast(final ParserRuleContext source, final Type from, final Type to, final boolean explicit) {
+        final Cast cast = new Cast(from, to);
+
+        if (from.equals(to)) {
+            return cast;
+        }
+
+        if (from.sort == Sort.DEF && to.sort != Sort.VOID || from.sort != Sort.VOID && to.sort == Sort.DEF) {
+            final Transform transform = definition.transforms.get(cast);
+
+            if (transform != null) {
+                return transform;
+            }
+
+            return cast;
+        }
+
+        switch (from.sort) {
+            case BOOL:
+                switch (to.sort) {
+                    case OBJECT:
+                    case BOOL_OBJ:
+                        return checkTransform(source, cast);
+                }
+
+                break;
+            case BYTE:
+                switch (to.sort) {
+                    case SHORT:
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                        return cast;
+                    case CHAR:
+                        if (explicit)
+                            return cast;
+
+                        break;
+                    case OBJECT:
+                    case NUMBER:
+                    case BYTE_OBJ:
+                    case SHORT_OBJ:
+                    case INT_OBJ:
+                    case LONG_OBJ:
+                    case FLOAT_OBJ:
+                    case DOUBLE_OBJ:
+                        return checkTransform(source, cast);
+                    case CHAR_OBJ:
+                        if (explicit)
+                            return checkTransform(source, cast);
+
+                        break;
+                }
+
+                break;
+            case SHORT:
+                switch (to.sort) {
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                        return cast;
+                    case BYTE:
+                    case CHAR:
+                        if (explicit)
+                            return cast;
+
+                        break;
+                    case OBJECT:
+                    case NUMBER:
+                    case SHORT_OBJ:
+                    case INT_OBJ:
+                    case LONG_OBJ:
+                    case FLOAT_OBJ:
+                    case DOUBLE_OBJ:
+                        return checkTransform(source, cast);
+                    case BYTE_OBJ:
+                    case CHAR_OBJ:
+                        if (explicit)
+                            return checkTransform(source, cast);
+
+                        break;
+                }
+
+                break;
+            case CHAR:
+                switch (to.sort) {
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                        return cast;
+                    case BYTE:
+                    case SHORT:
+                        if (explicit)
+                            return cast;
+
+                        break;
+                    case OBJECT:
+                    case NUMBER:
+                    case CHAR_OBJ:
+                    case INT_OBJ:
+                    case LONG_OBJ:
+                    case FLOAT_OBJ:
+                    case DOUBLE_OBJ:
+                        return checkTransform(source, cast);
+                    case BYTE_OBJ:
+                    case SHORT_OBJ:
+                        if (explicit)
+                            return checkTransform(source, cast);
+
+                        break;
+                }
+
+                break;
+            case INT:
+                switch (to.sort) {
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                        return cast;
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                        if (explicit)
+                            return cast;
+
+                        break;
+                    case OBJECT:
+                    case NUMBER:
+                    case INT_OBJ:
+                    case LONG_OBJ:
+                    case FLOAT_OBJ:
+                    case DOUBLE_OBJ:
+                        return checkTransform(source, cast);
+                    case BYTE_OBJ:
+                    case SHORT_OBJ:
+                    case CHAR_OBJ:
+                        if (explicit)
+                            return checkTransform(source, cast);
+
+                        break;
+                }
+
+                break;
+            case LONG:
+                switch (to.sort) {
+                    case FLOAT:
+                    case DOUBLE:
+                        return cast;
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                    case INT:
+                        if (explicit)
+                            return cast;
+
+                        break;
+                    case OBJECT:
+                    case NUMBER:
+                    case LONG_OBJ:
+                    case FLOAT_OBJ:
+                    case DOUBLE_OBJ:
+                        return checkTransform(source, cast);
+                    case BYTE_OBJ:
+                    case SHORT_OBJ:
+                    case CHAR_OBJ:
+                    case INT_OBJ:
+                        if (explicit)
+                            return checkTransform(source, cast);
+
+                        break;
+                }
+
+                break;
+            case FLOAT:
+                switch (to.sort) {
+                    case DOUBLE:
+                        return cast;
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                    case INT:
+                    case LONG:
+                        if (explicit)
+                            return cast;
+
+                        break;
+                    case OBJECT:
+                    case NUMBER:
+                    case FLOAT_OBJ:
+                    case DOUBLE_OBJ:
+                        return checkTransform(source, cast);
+                    case BYTE_OBJ:
+                    case SHORT_OBJ:
+                    case CHAR_OBJ:
+                    case INT_OBJ:
+                    case LONG_OBJ:
+                        if (explicit)
+                            return checkTransform(source, cast);
+
+                        break;
+                }
+
+                break;
+            case DOUBLE:
+                switch (to.sort) {
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                        if (explicit)
+                            return cast;
+
+                        break;
+                    case OBJECT:
+                    case NUMBER:
+                    case DOUBLE_OBJ:
+                        return checkTransform(source, cast);
+                    case BYTE_OBJ:
+                    case SHORT_OBJ:
+                    case CHAR_OBJ:
+                    case INT_OBJ:
+                    case LONG_OBJ:
+                    case FLOAT_OBJ:
+                        if (explicit)
+                            return checkTransform(source, cast);
+
+                        break;
+                }
+
+                break;
+            case OBJECT:
+            case NUMBER:
+                switch (to.sort) {
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                        if (explicit)
+                            return checkTransform(source, cast);
+
+                        break;
+                }
+
+                break;
+            case BOOL_OBJ:
+                switch (to.sort) {
+                    case BOOL:
+                        return checkTransform(source, cast);
+                }
+
+                break;
+            case BYTE_OBJ:
+                switch (to.sort) {
+                    case BYTE:
+                    case SHORT:
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                    case SHORT_OBJ:
+                    case INT_OBJ:
+                    case LONG_OBJ:
+                    case FLOAT_OBJ:
+                    case DOUBLE_OBJ:
+                        return checkTransform(source, cast);
+                    case CHAR:
+                    case CHAR_OBJ:
+                        if (explicit)
+                            return checkTransform(source, cast);
+
+                        break;
+                }
+
+                break;
+            case SHORT_OBJ:
+                switch (to.sort) {
+                    case SHORT:
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                    case INT_OBJ:
+                    case LONG_OBJ:
+                    case FLOAT_OBJ:
+                    case DOUBLE_OBJ:
+                        return checkTransform(source, cast);
+                    case BYTE:
+                    case CHAR:
+                    case BYTE_OBJ:
+                    case CHAR_OBJ:
+                        if (explicit)
+                            return checkTransform(source, cast);
+
+                        break;
+                }
+
+                break;
+            case CHAR_OBJ:
+                switch (to.sort) {
+                    case CHAR:
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                    case INT_OBJ:
+                    case LONG_OBJ:
+                    case FLOAT_OBJ:
+                    case DOUBLE_OBJ:
+                        return checkTransform(source, cast);
+                    case BYTE:
+                    case SHORT:
+                    case BYTE_OBJ:
+                    case SHORT_OBJ:
+                        if (explicit)
+                            return checkTransform(source, cast);
+
+                        break;
+                }
+
+                break;
+            case INT_OBJ:
+                switch (to.sort) {
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                    case LONG_OBJ:
+                    case FLOAT_OBJ:
+                    case DOUBLE_OBJ:
+                        return checkTransform(source, cast);
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                    case BYTE_OBJ:
+                    case SHORT_OBJ:
+                    case CHAR_OBJ:
+                        if (explicit)
+                            return checkTransform(source, cast);
+
+                        break;
+                }
+
+                break;
+            case LONG_OBJ:
+                switch (to.sort) {
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                    case FLOAT_OBJ:
+                    case DOUBLE_OBJ:
+                        return checkTransform(source, cast);
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                    case INT:
+                    case BYTE_OBJ:
+                    case SHORT_OBJ:
+                    case CHAR_OBJ:
+                    case INT_OBJ:
+                        if (explicit)
+                            return checkTransform(source, cast);
+
+                        break;
+                }
+
+                break;
+            case FLOAT_OBJ:
+                switch (to.sort) {
+                    case FLOAT:
+                    case DOUBLE:
+                    case DOUBLE_OBJ:
+                        return checkTransform(source, cast);
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                    case INT:
+                    case LONG:
+                    case BYTE_OBJ:
+                    case SHORT_OBJ:
+                    case CHAR_OBJ:
+                    case INT_OBJ:
+                    case LONG_OBJ:
+                        if (explicit)
+                            return checkTransform(source, cast);
+
+                        break;
+                }
+
+                break;
+            case DOUBLE_OBJ:
+                switch (to.sort) {
+                    case DOUBLE:
+                        return checkTransform(source, cast);
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                    case BYTE_OBJ:
+                    case SHORT_OBJ:
+                    case CHAR_OBJ:
+                    case INT_OBJ:
+                    case LONG_OBJ:
+                    case FLOAT_OBJ:
+                        if (explicit)
+                            return checkTransform(source, cast);
+
+                        break;
+                }
+
+                break;
+        }
+
+        try {
+            from.clazz.asSubclass(to.clazz);
+
+            return cast;
+        } catch (final ClassCastException cce0) {
+            try {
+                if (explicit) {
+                    to.clazz.asSubclass(from.clazz);
+
+                    return cast;
+                } else {
+                    throw new ClassCastException(
+                        AnalyzerUtility.error(source) + "Cannot cast from [" + from.name + "] to [" + to.name + "].");
+                }
+            } catch (final ClassCastException cce1) {
+                throw new ClassCastException(
+                    AnalyzerUtility.error(source) + "Cannot cast from [" + from.name + "] to [" + to.name + "].");
+            }
+        }
+    }
+
+    private Transform checkTransform(final ParserRuleContext source, final Cast cast) {
+        final Transform transform = definition.transforms.get(cast);
+
+        if (transform == null) {
+            throw new ClassCastException(
+                AnalyzerUtility.error(source) + "Cannot cast from [" + cast.from.name + "] to [" + cast.to.name + "].");
+        }
+
+        return transform;
+    }
+
+    private Object constCast(final ParserRuleContext source, final Object constant, final Cast cast) {
+        if (cast instanceof Transform) {
+            final Transform transform = (Transform)cast;
+            return invokeTransform(source, transform, constant);
+        } else {
+            final Sort fsort = cast.from.sort;
+            final Sort tsort = cast.to.sort;
+
+            if (fsort == tsort) {
+                return constant;
+            } else if (fsort.numeric && tsort.numeric) {
+                Number number;
+
+                if (fsort == Sort.CHAR) {
+                    number = (int)(char)constant;
+                } else {
+                    number = (Number)constant;
+                }
+
+                switch (tsort) {
+                    case BYTE:   return number.byteValue();
+                    case SHORT:  return number.shortValue();
+                    case CHAR:   return (char)number.intValue();
+                    case INT:    return number.intValue();
+                    case LONG:   return number.longValue();
+                    case FLOAT:  return number.floatValue();
+                    case DOUBLE: return number.doubleValue();
+                    default:
+                        throw new IllegalStateException(AnalyzerUtility.error(source) + "Expected numeric type for cast.");
+                }
+            } else {
+                throw new IllegalStateException(AnalyzerUtility.error(source) + "No valid constant cast from " +
+                    "[" + cast.from.clazz.getCanonicalName() + "] to " +
+                    "[" + cast.to.clazz.getCanonicalName() + "].");
+            }
+        }
+    }
+
+    private Object invokeTransform(final ParserRuleContext source, final Transform transform, final Object object) {
+        final Method method = transform.method;
+        final java.lang.reflect.Method jmethod = method.reflect;
+        final int modifiers = jmethod.getModifiers();
+
+        try {
+            if (java.lang.reflect.Modifier.isStatic(modifiers)) {
+                return jmethod.invoke(null, object);
+            } else {
+                return jmethod.invoke(object);
+            }
+        } catch (IllegalAccessException | IllegalArgumentException |
+            java.lang.reflect.InvocationTargetException | NullPointerException |
+            ExceptionInInitializerError exception) {
+            throw new IllegalStateException(AnalyzerUtility.error(source) + "Unable to invoke transform to cast constant from " +
+                "[" + transform.from.name + "] to [" + transform.to.name + "].");
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExpression.java
new file mode 100644
index 00000000000..3e74259fecf
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExpression.java
@@ -0,0 +1,868 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Metadata.ExpressionMetadata;
+import org.elasticsearch.painless.Metadata.ExternalMetadata;
+import org.elasticsearch.painless.PainlessParser.AssignmentContext;
+import org.elasticsearch.painless.PainlessParser.BinaryContext;
+import org.elasticsearch.painless.PainlessParser.BoolContext;
+import org.elasticsearch.painless.PainlessParser.CastContext;
+import org.elasticsearch.painless.PainlessParser.CharContext;
+import org.elasticsearch.painless.PainlessParser.CompContext;
+import org.elasticsearch.painless.PainlessParser.ConditionalContext;
+import org.elasticsearch.painless.PainlessParser.DecltypeContext;
+import org.elasticsearch.painless.PainlessParser.ExpressionContext;
+import org.elasticsearch.painless.PainlessParser.ExternalContext;
+import org.elasticsearch.painless.PainlessParser.ExtstartContext;
+import org.elasticsearch.painless.PainlessParser.FalseContext;
+import org.elasticsearch.painless.PainlessParser.IncrementContext;
+import org.elasticsearch.painless.PainlessParser.NullContext;
+import org.elasticsearch.painless.PainlessParser.NumericContext;
+import org.elasticsearch.painless.PainlessParser.PostincContext;
+import org.elasticsearch.painless.PainlessParser.PreincContext;
+import org.elasticsearch.painless.PainlessParser.TrueContext;
+import org.elasticsearch.painless.PainlessParser.UnaryContext;
+
+import static org.elasticsearch.painless.PainlessParser.ADD;
+import static org.elasticsearch.painless.PainlessParser.BWAND;
+import static org.elasticsearch.painless.PainlessParser.BWOR;
+import static org.elasticsearch.painless.PainlessParser.BWXOR;
+import static org.elasticsearch.painless.PainlessParser.DIV;
+import static org.elasticsearch.painless.PainlessParser.LSH;
+import static org.elasticsearch.painless.PainlessParser.MUL;
+import static org.elasticsearch.painless.PainlessParser.REM;
+import static org.elasticsearch.painless.PainlessParser.RSH;
+import static org.elasticsearch.painless.PainlessParser.SUB;
+import static org.elasticsearch.painless.PainlessParser.USH;
+
+class AnalyzerExpression {
+    private final Metadata metadata;
+    private final Definition definition;
+    private final CompilerSettings settings;
+
+    private final Analyzer analyzer;
+    private final AnalyzerCaster caster;
+    private final AnalyzerPromoter promoter;
+
+    AnalyzerExpression(final Metadata metadata, final Analyzer analyzer,
+                       final AnalyzerCaster caster, final AnalyzerPromoter promoter) {
+        this.metadata = metadata;
+        this.definition = metadata.definition;
+        this.settings = metadata.settings;
+
+        this.analyzer = analyzer;
+        this.caster = caster;
+        this.promoter = promoter;
+    }
+
+    void processNumeric(final NumericContext ctx) {
+        final ExpressionMetadata numericemd = metadata.getExpressionMetadata(ctx);
+        final boolean negate = ctx.parent instanceof UnaryContext && ((UnaryContext)ctx.parent).SUB() != null;
+
+        if (ctx.DECIMAL() != null) {
+            final String svalue = (negate ? "-" : "") + ctx.DECIMAL().getText();
+
+            if (svalue.endsWith("f") || svalue.endsWith("F")) {
+                try {
+                    numericemd.from = definition.floatType;
+                    numericemd.preConst = Float.parseFloat(svalue.substring(0, svalue.length() - 1));
+                } catch (NumberFormatException exception) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Invalid float constant [" + svalue + "].");
+                }
+            } else {
+                try {
+                    numericemd.from = definition.doubleType;
+                    numericemd.preConst = Double.parseDouble(svalue);
+                } catch (NumberFormatException exception) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Invalid double constant [" + svalue + "].");
+                }
+            }
+        } else {
+            String svalue = negate ? "-" : "";
+            int radix;
+
+            if (ctx.OCTAL() != null) {
+                svalue += ctx.OCTAL().getText();
+                radix = 8;
+            } else if (ctx.INTEGER() != null) {
+                svalue += ctx.INTEGER().getText();
+                radix = 10;
+            } else if (ctx.HEX() != null) {
+                svalue += ctx.HEX().getText();
+                radix = 16;
+            } else {
+                throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+            }
+
+            if (svalue.endsWith("d") || svalue.endsWith("D")) {
+                try {
+                    numericemd.from = definition.doubleType;
+                    numericemd.preConst = Double.parseDouble(svalue.substring(0, svalue.length() - 1));
+                } catch (NumberFormatException exception) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Invalid float constant [" + svalue + "].");
+                }
+            } else if (svalue.endsWith("f") || svalue.endsWith("F")) {
+                try {
+                    numericemd.from = definition.floatType;
+                    numericemd.preConst = Float.parseFloat(svalue.substring(0, svalue.length() - 1));
+                } catch (NumberFormatException exception) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Invalid float constant [" + svalue + "].");
+                }
+            } else if (svalue.endsWith("l") || svalue.endsWith("L")) {
+                try {
+                    numericemd.from = definition.longType;
+                    numericemd.preConst = Long.parseLong(svalue.substring(0, svalue.length() - 1), radix);
+                } catch (NumberFormatException exception) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Invalid long constant [" + svalue + "].");
+                }
+            } else {
+                try {
+                    final Type type = numericemd.to;
+                    final Sort sort = type == null ? Sort.INT : type.sort;
+                    final int value = Integer.parseInt(svalue, radix);
+
+                    if (sort == Sort.BYTE && value >= Byte.MIN_VALUE && value <= Byte.MAX_VALUE) {
+                        numericemd.from = definition.byteType;
+                        numericemd.preConst = (byte)value;
+                    } else if (sort == Sort.CHAR && value >= Character.MIN_VALUE && value <= Character.MAX_VALUE) {
+                        numericemd.from = definition.charType;
+                        numericemd.preConst = (char)value;
+                    } else if (sort == Sort.SHORT && value >= Short.MIN_VALUE && value <= Short.MAX_VALUE) {
+                        numericemd.from = definition.shortType;
+                        numericemd.preConst = (short)value;
+                    } else {
+                        numericemd.from = definition.intType;
+                        numericemd.preConst = value;
+                    }
+                } catch (NumberFormatException exception) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Invalid int constant [" + svalue + "].");
+                }
+            }
+        }
+    }
+
+    void processChar(final CharContext ctx) {
+        final ExpressionMetadata charemd = metadata.getExpressionMetadata(ctx);
+
+        if (ctx.CHAR() == null) {
+            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+        }
+
+        charemd.preConst = ctx.CHAR().getText().charAt(0);
+        charemd.from = definition.charType;
+    }
+
+    void processTrue(final TrueContext ctx) {
+        final ExpressionMetadata trueemd = metadata.getExpressionMetadata(ctx);
+
+        if (ctx.TRUE() == null) {
+            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+        }
+
+        trueemd.preConst = true;
+        trueemd.from = definition.booleanType;
+    }
+
+    void processFalse(final FalseContext ctx) {
+        final ExpressionMetadata falseemd = metadata.getExpressionMetadata(ctx);
+
+        if (ctx.FALSE() == null) {
+            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+        }
+
+        falseemd.preConst = false;
+        falseemd.from = definition.booleanType;
+    }
+
+    void processNull(final NullContext ctx) {
+        final ExpressionMetadata nullemd = metadata.getExpressionMetadata(ctx);
+
+        if (ctx.NULL() == null) {
+            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+        }
+
+        nullemd.isNull = true;
+
+        if (nullemd.to != null) {
+            if (nullemd.to.sort.primitive) {
+                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
+                    "Cannot cast null to a primitive type [" + nullemd.to.name + "].");
+            }
+
+            nullemd.from = nullemd.to;
+        } else {
+            nullemd.from = definition.objectType;
+        }
+    }
+
+    void processExternal(final ExternalContext ctx) {
+        final ExpressionMetadata extemd = metadata.getExpressionMetadata(ctx);
+
+        final ExtstartContext extstartctx = ctx.extstart();
+        final ExternalMetadata extstartemd = metadata.createExternalMetadata(extstartctx);
+        extstartemd.read = extemd.read;
+        analyzer.visit(extstartctx);
+
+        extemd.statement = extstartemd.statement;
+        extemd.preConst = extstartemd.constant;
+        extemd.from = extstartemd.current;
+        extemd.typesafe = extstartemd.current.sort != Sort.DEF;
+    }
+
+    void processPostinc(final PostincContext ctx) {
+        final ExpressionMetadata postincemd = metadata.getExpressionMetadata(ctx);
+
+        final ExtstartContext extstartctx = ctx.extstart();
+        final ExternalMetadata extstartemd = metadata.createExternalMetadata(extstartctx);
+        extstartemd.read = postincemd.read;
+        extstartemd.storeExpr = ctx.increment();
+        extstartemd.token = ADD;
+        extstartemd.post = true;
+        analyzer.visit(extstartctx);
+
+        postincemd.statement = true;
+        postincemd.from = extstartemd.read ? extstartemd.current : definition.voidType;
+        postincemd.typesafe = extstartemd.current.sort != Sort.DEF;
+    }
+
+    void processPreinc(final PreincContext ctx) {
+        final ExpressionMetadata preincemd = metadata.getExpressionMetadata(ctx);
+
+        final ExtstartContext extstartctx = ctx.extstart();
+        final ExternalMetadata extstartemd = metadata.createExternalMetadata(extstartctx);
+        extstartemd.read = preincemd.read;
+        extstartemd.storeExpr = ctx.increment();
+        extstartemd.token = ADD;
+        extstartemd.pre = true;
+        analyzer.visit(extstartctx);
+
+        preincemd.statement = true;
+        preincemd.from = extstartemd.read ? extstartemd.current : definition.voidType;
+        preincemd.typesafe = extstartemd.current.sort != Sort.DEF;
+    }
+
+    void processUnary(final UnaryContext ctx) {
+        final ExpressionMetadata unaryemd = metadata.getExpressionMetadata(ctx);
+
+        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
+        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
+
+        if (ctx.BOOLNOT() != null) {
+            expremd.to = definition.booleanType;
+            analyzer.visit(exprctx);
+            caster.markCast(expremd);
+
+            if (expremd.postConst != null) {
+                unaryemd.preConst = !(boolean)expremd.postConst;
+            }
+
+            unaryemd.from = definition.booleanType;
+        } else if (ctx.BWNOT() != null || ctx.ADD() != null || ctx.SUB() != null) {
+            analyzer.visit(exprctx);
+
+            final Type promote = promoter.promoteNumeric(expremd.from, ctx.BWNOT() == null, true);
+
+            if (promote == null) {
+                throw new ClassCastException(AnalyzerUtility.error(ctx) + "Cannot apply [" + ctx.getChild(0).getText() + "] " +
+                    "operation to type [" + expremd.from.name + "].");
+            }
+
+            expremd.to = promote;
+            caster.markCast(expremd);
+
+            if (expremd.postConst != null) {
+                final Sort sort = promote.sort;
+
+                if (ctx.BWNOT() != null) {
+                    if (sort == Sort.INT) {
+                        unaryemd.preConst = ~(int)expremd.postConst;
+                    } else if (sort == Sort.LONG) {
+                        unaryemd.preConst = ~(long)expremd.postConst;
+                    } else {
+                        throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                    }
+                } else if (ctx.SUB() != null) {
+                    if (exprctx instanceof NumericContext) {
+                        unaryemd.preConst = expremd.postConst;
+                    } else {
+                        if (sort == Sort.INT) {
+                            if (settings.getNumericOverflow()) {
+                                unaryemd.preConst = -(int)expremd.postConst;
+                            } else {
+                                unaryemd.preConst = Math.negateExact((int)expremd.postConst);
+                            }
+                        } else if (sort == Sort.LONG) {
+                            if (settings.getNumericOverflow()) {
+                                unaryemd.preConst = -(long)expremd.postConst;
+                            } else {
+                                unaryemd.preConst = Math.negateExact((long)expremd.postConst);
+                            }
+                        } else if (sort == Sort.FLOAT) {
+                            unaryemd.preConst = -(float)expremd.postConst;
+                        } else if (sort == Sort.DOUBLE) {
+                            unaryemd.preConst = -(double)expremd.postConst;
+                        } else {
+                            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                        }
+                    }
+                } else if (ctx.ADD() != null) {
+                    if (sort == Sort.INT) {
+                        unaryemd.preConst = +(int)expremd.postConst;
+                    } else if (sort == Sort.LONG) {
+                        unaryemd.preConst = +(long)expremd.postConst;
+                    } else if (sort == Sort.FLOAT) {
+                        unaryemd.preConst = +(float)expremd.postConst;
+                    } else if (sort == Sort.DOUBLE) {
+                        unaryemd.preConst = +(double)expremd.postConst;
+                    } else {
+                        throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                    }
+                } else {
+                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                }
+            }
+
+            unaryemd.from = promote;
+            unaryemd.typesafe = expremd.typesafe;
+        } else {
+            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+        }
+    }
+
+    void processCast(final CastContext ctx) {
+        final ExpressionMetadata castemd = metadata.getExpressionMetadata(ctx);
+
+        final DecltypeContext decltypectx = ctx.decltype();
+        final ExpressionMetadata decltypemd = metadata.createExpressionMetadata(decltypectx);
+        analyzer.visit(decltypectx);
+
+        final Type type = decltypemd.from;
+        castemd.from = type;
+
+        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
+        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
+        expremd.to = type;
+        expremd.explicit = true;
+        analyzer.visit(exprctx);
+        caster.markCast(expremd);
+
+        if (expremd.postConst != null) {
+            castemd.preConst = expremd.postConst;
+        }
+
+        castemd.typesafe = expremd.typesafe && castemd.from.sort != Sort.DEF;
+    }
+
+    void processBinary(final BinaryContext ctx) {
+        final ExpressionMetadata binaryemd = metadata.getExpressionMetadata(ctx);
+
+        final ExpressionContext exprctx0 = AnalyzerUtility.updateExpressionTree(ctx.expression(0));
+        final ExpressionMetadata expremd0 = metadata.createExpressionMetadata(exprctx0);
+        analyzer.visit(exprctx0);
+
+        final ExpressionContext exprctx1 = AnalyzerUtility.updateExpressionTree(ctx.expression(1));
+        final ExpressionMetadata expremd1 = metadata.createExpressionMetadata(exprctx1);
+        analyzer.visit(exprctx1);
+
+        final boolean decimal = ctx.MUL() != null || ctx.DIV() != null || ctx.REM() != null || ctx.SUB() != null;
+        final boolean add = ctx.ADD() != null;
+        final boolean xor = ctx.BWXOR() != null;
+        final Type promote = add ? promoter.promoteAdd(expremd0.from, expremd1.from) :
+            xor ? promoter.promoteXor(expremd0.from, expremd1.from) :
+                promoter.promoteNumeric(expremd0.from, expremd1.from, decimal, true);
+
+        if (promote == null) {
+            throw new ClassCastException(AnalyzerUtility.error(ctx) + "Cannot apply [" + ctx.getChild(1).getText() + "] " +
+                "operation to types [" + expremd0.from.name + "] and [" + expremd1.from.name + "].");
+        }
+
+        final Sort sort = promote.sort;
+        expremd0.to = add && sort == Sort.STRING ? expremd0.from : promote;
+        expremd1.to = add && sort == Sort.STRING ? expremd1.from : promote;
+        caster.markCast(expremd0);
+        caster.markCast(expremd1);
+
+        if (expremd0.postConst != null && expremd1.postConst != null) {
+            if (ctx.MUL() != null) {
+                if (sort == Sort.INT) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (int)expremd0.postConst * (int)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Math.multiplyExact((int)expremd0.postConst, (int)expremd1.postConst);
+                    }
+                } else if (sort == Sort.LONG) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (long)expremd0.postConst * (long)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Math.multiplyExact((long)expremd0.postConst, (long)expremd1.postConst);
+                    }
+                } else if (sort == Sort.FLOAT) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (float)expremd0.postConst * (float)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Utility.multiplyWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst);
+                    }
+                } else if (sort == Sort.DOUBLE) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (double)expremd0.postConst * (double)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Utility.multiplyWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst);
+                    }
+                } else {
+                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                }
+            } else if (ctx.DIV() != null) {
+                if (sort == Sort.INT) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (int)expremd0.postConst / (int)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Utility.divideWithoutOverflow((int)expremd0.postConst, (int)expremd1.postConst);
+                    }
+                } else if (sort == Sort.LONG) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (long)expremd0.postConst / (long)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Utility.divideWithoutOverflow((long)expremd0.postConst, (long)expremd1.postConst);
+                    }
+                } else if (sort == Sort.FLOAT) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (float)expremd0.postConst / (float)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Utility.divideWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst);
+                    }
+                } else if (sort == Sort.DOUBLE) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (double)expremd0.postConst / (double)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Utility.divideWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst);
+                    }
+                } else {
+                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                }
+            } else if (ctx.REM() != null) {
+                if (sort == Sort.INT) {
+                    binaryemd.preConst = (int)expremd0.postConst % (int)expremd1.postConst;
+                } else if (sort == Sort.LONG) {
+                    binaryemd.preConst = (long)expremd0.postConst % (long)expremd1.postConst;
+                } else if (sort == Sort.FLOAT) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (float)expremd0.postConst % (float)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Utility.remainderWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst);
+                    }
+                } else if (sort == Sort.DOUBLE) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (double)expremd0.postConst % (double)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Utility.remainderWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst);
+                    }
+                } else {
+                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                }
+            } else if (ctx.ADD() != null) {
+                if (sort == Sort.INT) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (int)expremd0.postConst + (int)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Math.addExact((int)expremd0.postConst, (int)expremd1.postConst);
+                    }
+                } else if (sort == Sort.LONG) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (long)expremd0.postConst + (long)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Math.addExact((long)expremd0.postConst, (long)expremd1.postConst);
+                    }
+                } else if (sort == Sort.FLOAT) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (float)expremd0.postConst + (float)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Utility.addWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst);
+                    }
+                } else if (sort == Sort.DOUBLE) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (double)expremd0.postConst + (double)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Utility.addWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst);
+                    }
+                } else if (sort == Sort.STRING) {
+                    binaryemd.preConst = "" + expremd0.postConst + expremd1.postConst;
+                } else {
+                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                }
+            } else if (ctx.SUB() != null) {
+                if (sort == Sort.INT) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (int)expremd0.postConst - (int)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Math.subtractExact((int)expremd0.postConst, (int)expremd1.postConst);
+                    }
+                } else if (sort == Sort.LONG) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (long)expremd0.postConst - (long)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Math.subtractExact((long)expremd0.postConst, (long)expremd1.postConst);
+                    }
+                } else if (sort == Sort.FLOAT) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (float)expremd0.postConst - (float)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Utility.subtractWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst);
+                    }
+                } else if (sort == Sort.DOUBLE) {
+                    if (settings.getNumericOverflow()) {
+                        binaryemd.preConst = (double)expremd0.postConst - (double)expremd1.postConst;
+                    } else {
+                        binaryemd.preConst = Utility.subtractWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst);
+                    }
+                } else {
+                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                }
+            } else if (ctx.LSH() != null) {
+                if (sort == Sort.INT) {
+                    binaryemd.preConst = (int)expremd0.postConst << (int)expremd1.postConst;
+                } else if (sort == Sort.LONG) {
+                    binaryemd.preConst = (long)expremd0.postConst << (long)expremd1.postConst;
+                } else {
+                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                }
+            } else if (ctx.RSH() != null) {
+                if (sort == Sort.INT) {
+                    binaryemd.preConst = (int)expremd0.postConst >> (int)expremd1.postConst;
+                } else if (sort == Sort.LONG) {
+                    binaryemd.preConst = (long)expremd0.postConst >> (long)expremd1.postConst;
+                } else {
+                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                }
+            } else if (ctx.USH() != null) {
+                if (sort == Sort.INT) {
+                    binaryemd.preConst = (int)expremd0.postConst >>> (int)expremd1.postConst;
+                } else if (sort == Sort.LONG) {
+                    binaryemd.preConst = (long)expremd0.postConst >>> (long)expremd1.postConst;
+                } else {
+                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                }
+            } else if (ctx.BWAND() != null) {
+                if (sort == Sort.INT) {
+                    binaryemd.preConst = (int)expremd0.postConst & (int)expremd1.postConst;
+                } else if (sort == Sort.LONG) {
+                    binaryemd.preConst = (long)expremd0.postConst & (long)expremd1.postConst;
+                } else {
+                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                }
+            } else if (ctx.BWXOR() != null) {
+                if (sort == Sort.BOOL) {
+                    binaryemd.preConst = (boolean)expremd0.postConst ^ (boolean)expremd1.postConst;
+                } else if (sort == Sort.INT) {
+                    binaryemd.preConst = (int)expremd0.postConst ^ (int)expremd1.postConst;
+                } else if (sort == Sort.LONG) {
+                    binaryemd.preConst = (long)expremd0.postConst ^ (long)expremd1.postConst;
+                } else {
+                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                }
+            } else if (ctx.BWOR() != null) {
+                if (sort == Sort.INT) {
+                    binaryemd.preConst = (int)expremd0.postConst | (int)expremd1.postConst;
+                } else if (sort == Sort.LONG) {
+                    binaryemd.preConst = (long)expremd0.postConst | (long)expremd1.postConst;
+                } else {
+                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                }
+            } else {
+                throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+            }
+        }
+
+        binaryemd.from = promote;
+        binaryemd.typesafe = expremd0.typesafe && expremd1.typesafe;
+    }
+
+    void processComp(final CompContext ctx) {
+        final ExpressionMetadata compemd = metadata.getExpressionMetadata(ctx);
+        final boolean equality = ctx.EQ() != null || ctx.NE() != null;
+        final boolean reference = ctx.EQR() != null || ctx.NER() != null;
+
+        final ExpressionContext exprctx0 = AnalyzerUtility.updateExpressionTree(ctx.expression(0));
+        final ExpressionMetadata expremd0 = metadata.createExpressionMetadata(exprctx0);
+        analyzer.visit(exprctx0);
+
+        final ExpressionContext exprctx1 = AnalyzerUtility.updateExpressionTree(ctx.expression(1));
+        final ExpressionMetadata expremd1 = metadata.createExpressionMetadata(exprctx1);
+        analyzer.visit(exprctx1);
+
+        if (expremd0.isNull && expremd1.isNull) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unnecessary comparison of null constants.");
+        }
+
+        final Type promote = equality ? promoter.promoteEquality(expremd0.from, expremd1.from) :
+            reference ? promoter.promoteReference(expremd0.from, expremd1.from) :
+                promoter.promoteNumeric(expremd0.from, expremd1.from, true, true);
+
+        if (promote == null) {
+            throw new ClassCastException(AnalyzerUtility.error(ctx) + "Cannot apply [" + ctx.getChild(1).getText() + "] " +
+                "operation to types [" + expremd0.from.name + "] and [" + expremd1.from.name + "].");
+        }
+
+        expremd0.to = promote;
+        expremd1.to = promote;
+        caster.markCast(expremd0);
+        caster.markCast(expremd1);
+
+        if (expremd0.postConst != null && expremd1.postConst != null) {
+            final Sort sort = promote.sort;
+
+            if (ctx.EQ() != null || ctx.EQR() != null) {
+                if (sort == Sort.BOOL) {
+                    compemd.preConst = (boolean)expremd0.postConst == (boolean)expremd1.postConst;
+                } else if (sort == Sort.INT) {
+                    compemd.preConst = (int)expremd0.postConst == (int)expremd1.postConst;
+                } else if (sort == Sort.LONG) {
+                    compemd.preConst = (long)expremd0.postConst == (long)expremd1.postConst;
+                } else if (sort == Sort.FLOAT) {
+                    compemd.preConst = (float)expremd0.postConst == (float)expremd1.postConst;
+                } else if (sort == Sort.DOUBLE) {
+                    compemd.preConst = (double)expremd0.postConst == (double)expremd1.postConst;
+                } else {
+                    if (ctx.EQ() != null && !expremd0.isNull && !expremd1.isNull) {
+                        compemd.preConst = expremd0.postConst.equals(expremd1.postConst);
+                    } else if (ctx.EQR() != null) {
+                        compemd.preConst = expremd0.postConst == expremd1.postConst;
+                    } else {
+                        throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                    }
+                }
+            } else if (ctx.NE() != null || ctx.NER() != null) {
+                if (sort == Sort.BOOL) {
+                    compemd.preConst = (boolean)expremd0.postConst != (boolean)expremd1.postConst;
+                } else if (sort == Sort.INT) {
+                    compemd.preConst = (int)expremd0.postConst != (int)expremd1.postConst;
+                } else if (sort == Sort.LONG) {
+                    compemd.preConst = (long)expremd0.postConst != (long)expremd1.postConst;
+                } else if (sort == Sort.FLOAT) {
+                    compemd.preConst = (float)expremd0.postConst != (float)expremd1.postConst;
+                } else if (sort == Sort.DOUBLE) {
+                    compemd.preConst = (double)expremd0.postConst != (double)expremd1.postConst;
+                } else {
+                    if (ctx.NE() != null && !expremd0.isNull && !expremd1.isNull) {
+                        compemd.preConst = expremd0.postConst.equals(expremd1.postConst);
+                    } else if (ctx.NER() != null) {
+                        compemd.preConst = expremd0.postConst == expremd1.postConst;
+                    } else {
+                        throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                    }
+                }
+            } else if (ctx.GTE() != null) {
+                if (sort == Sort.INT) {
+                    compemd.preConst = (int)expremd0.postConst >= (int)expremd1.postConst;
+                } else if (sort == Sort.LONG) {
+                    compemd.preConst = (long)expremd0.postConst >= (long)expremd1.postConst;
+                } else if (sort == Sort.FLOAT) {
+                    compemd.preConst = (float)expremd0.postConst >= (float)expremd1.postConst;
+                } else if (sort == Sort.DOUBLE) {
+                    compemd.preConst = (double)expremd0.postConst >= (double)expremd1.postConst;
+                } else {
+                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                }
+            } else if (ctx.GT() != null) {
+                if (sort == Sort.INT) {
+                    compemd.preConst = (int)expremd0.postConst > (int)expremd1.postConst;
+                } else if (sort == Sort.LONG) {
+                    compemd.preConst = (long)expremd0.postConst > (long)expremd1.postConst;
+                } else if (sort == Sort.FLOAT) {
+                    compemd.preConst = (float)expremd0.postConst > (float)expremd1.postConst;
+                } else if (sort == Sort.DOUBLE) {
+                    compemd.preConst = (double)expremd0.postConst > (double)expremd1.postConst;
+                } else {
+                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                }
+            } else if (ctx.LTE() != null) {
+                if (sort == Sort.INT) {
+                    compemd.preConst = (int)expremd0.postConst <= (int)expremd1.postConst;
+                } else if (sort == Sort.LONG) {
+                    compemd.preConst = (long)expremd0.postConst <= (long)expremd1.postConst;
+                } else if (sort == Sort.FLOAT) {
+                    compemd.preConst = (float)expremd0.postConst <= (float)expremd1.postConst;
+                } else if (sort == Sort.DOUBLE) {
+                    compemd.preConst = (double)expremd0.postConst <= (double)expremd1.postConst;
+                } else {
+                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                }
+            } else if (ctx.LT() != null) {
+                if (sort == Sort.INT) {
+                    compemd.preConst = (int)expremd0.postConst < (int)expremd1.postConst;
+                } else if (sort == Sort.LONG) {
+                    compemd.preConst = (long)expremd0.postConst < (long)expremd1.postConst;
+                } else if (sort == Sort.FLOAT) {
+                    compemd.preConst = (float)expremd0.postConst < (float)expremd1.postConst;
+                } else if (sort == Sort.DOUBLE) {
+                    compemd.preConst = (double)expremd0.postConst < (double)expremd1.postConst;
+                } else {
+                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                }
+            } else {
+                throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+            }
+        }
+
+        compemd.from = definition.booleanType;
+        compemd.typesafe = expremd0.typesafe && expremd1.typesafe;
+    }
+
+    void processBool(final BoolContext ctx) {
+        final ExpressionMetadata boolemd = metadata.getExpressionMetadata(ctx);
+
+        final ExpressionContext exprctx0 = AnalyzerUtility.updateExpressionTree(ctx.expression(0));
+        final ExpressionMetadata expremd0 = metadata.createExpressionMetadata(exprctx0);
+        expremd0.to = definition.booleanType;
+        analyzer.visit(exprctx0);
+        caster.markCast(expremd0);
+
+        final ExpressionContext exprctx1 = AnalyzerUtility.updateExpressionTree(ctx.expression(1));
+        final ExpressionMetadata expremd1 = metadata.createExpressionMetadata(exprctx1);
+        expremd1.to = definition.booleanType;
+        analyzer.visit(exprctx1);
+        caster.markCast(expremd1);
+
+        if (expremd0.postConst != null && expremd1.postConst != null) {
+            if (ctx.BOOLAND() != null) {
+                boolemd.preConst = (boolean)expremd0.postConst && (boolean)expremd1.postConst;
+            } else if (ctx.BOOLOR() != null) {
+                boolemd.preConst = (boolean)expremd0.postConst || (boolean)expremd1.postConst;
+            } else {
+                throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+            }
+        }
+
+        boolemd.from = definition.booleanType;
+        boolemd.typesafe = expremd0.typesafe && expremd1.typesafe;
+    }
+
+    void processConditional(final ConditionalContext ctx) {
+        final ExpressionMetadata condemd = metadata.getExpressionMetadata(ctx);
+
+        final ExpressionContext exprctx0 = AnalyzerUtility.updateExpressionTree(ctx.expression(0));
+        final ExpressionMetadata expremd0 = metadata.createExpressionMetadata(exprctx0);
+        expremd0.to = definition.booleanType;
+        analyzer.visit(exprctx0);
+        caster.markCast(expremd0);
+
+        if (expremd0.postConst != null) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unnecessary conditional statement.");
+        }
+
+        final ExpressionContext exprctx1 = AnalyzerUtility.updateExpressionTree(ctx.expression(1));
+        final ExpressionMetadata expremd1 = metadata.createExpressionMetadata(exprctx1);
+        expremd1.to = condemd.to;
+        expremd1.explicit = condemd.explicit;
+        analyzer.visit(exprctx1);
+
+        final ExpressionContext exprctx2 = AnalyzerUtility.updateExpressionTree(ctx.expression(2));
+        final ExpressionMetadata expremd2 = metadata.createExpressionMetadata(exprctx2);
+        expremd2.to = condemd.to;
+        expremd2.explicit = condemd.explicit;
+        analyzer.visit(exprctx2);
+
+        if (condemd.to == null) {
+            final Type promote = promoter.promoteConditional(expremd1.from, expremd2.from, expremd1.preConst, expremd2.preConst);
+
+            expremd1.to = promote;
+            expremd2.to = promote;
+            condemd.from = promote;
+        } else {
+            condemd.from = condemd.to;
+        }
+
+        caster.markCast(expremd1);
+        caster.markCast(expremd2);
+
+        condemd.typesafe = expremd0.typesafe && expremd1.typesafe;
+    }
+
+    void processAssignment(final AssignmentContext ctx) {
+        final ExpressionMetadata assignemd = metadata.getExpressionMetadata(ctx);
+
+        final ExtstartContext extstartctx = ctx.extstart();
+        final ExternalMetadata extstartemd = metadata.createExternalMetadata(extstartctx);
+
+        extstartemd.read = assignemd.read;
+        extstartemd.storeExpr = AnalyzerUtility.updateExpressionTree(ctx.expression());
+
+        if (ctx.AMUL() != null) {
+            extstartemd.token = MUL;
+        } else if (ctx.ADIV() != null) {
+            extstartemd.token = DIV;
+        } else if (ctx.AREM() != null) {
+            extstartemd.token = REM;
+        } else if (ctx.AADD() != null) {
+            extstartemd.token = ADD;
+        } else if (ctx.ASUB() != null) {
+            extstartemd.token = SUB;
+        } else if (ctx.ALSH() != null) {
+            extstartemd.token = LSH;
+        } else if (ctx.AUSH() != null) {
+            extstartemd.token = USH;
+        } else if (ctx.ARSH() != null) {
+            extstartemd.token = RSH;
+        } else if (ctx.AAND() != null) {
+            extstartemd.token = BWAND;
+        } else if (ctx.AXOR() != null) {
+            extstartemd.token = BWXOR;
+        } else if (ctx.AOR() != null) {
+            extstartemd.token = BWOR;
+        }
+
+        analyzer.visit(extstartctx);
+
+        assignemd.statement = true;
+        assignemd.from = extstartemd.read ? extstartemd.current : definition.voidType;
+        assignemd.typesafe = extstartemd.current.sort != Sort.DEF;
+    }
+
+    void processIncrement(final IncrementContext ctx) {
+        final ExpressionMetadata incremd = metadata.getExpressionMetadata(ctx);
+        final Sort sort = incremd.to == null ? null : incremd.to.sort;
+        final boolean positive = ctx.INCR() != null;
+
+        if (incremd.to == null) {
+            incremd.preConst = positive ? 1 : -1;
+            incremd.from = definition.intType;
+        } else {
+            switch (sort) {
+                case LONG:
+                    incremd.preConst = positive ? 1L : -1L;
+                    incremd.from = definition.longType;
+                    break;
+                case FLOAT:
+                    incremd.preConst = positive ? 1.0F : -1.0F;
+                    incremd.from = definition.floatType;
+                    break;
+                case DOUBLE:
+                    incremd.preConst = positive ? 1.0 : -1.0;
+                    incremd.from = definition.doubleType;
+                    break;
+                default:
+                    incremd.preConst = positive ? 1 : -1;
+                    incremd.from = definition.intType;
+            }
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
new file mode 100644
index 00000000000..db3ab06e785
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
@@ -0,0 +1,816 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.elasticsearch.painless.AnalyzerUtility.Variable;
+import org.elasticsearch.painless.Definition.Constructor;
+import org.elasticsearch.painless.Definition.Field;
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Definition.Struct;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Metadata.ExpressionMetadata;
+import org.elasticsearch.painless.Metadata.ExtNodeMetadata;
+import org.elasticsearch.painless.Metadata.ExternalMetadata;
+import org.elasticsearch.painless.PainlessParser.DecltypeContext;
+import org.elasticsearch.painless.PainlessParser.ExpressionContext;
+import org.elasticsearch.painless.PainlessParser.ExtbraceContext;
+import org.elasticsearch.painless.PainlessParser.ExtcallContext;
+import org.elasticsearch.painless.PainlessParser.ExtcastContext;
+import org.elasticsearch.painless.PainlessParser.ExtdotContext;
+import org.elasticsearch.painless.PainlessParser.ExtfieldContext;
+import org.elasticsearch.painless.PainlessParser.ExtnewContext;
+import org.elasticsearch.painless.PainlessParser.ExtprecContext;
+import org.elasticsearch.painless.PainlessParser.ExtstartContext;
+import org.elasticsearch.painless.PainlessParser.ExtstringContext;
+import org.elasticsearch.painless.PainlessParser.ExttypeContext;
+import org.elasticsearch.painless.PainlessParser.ExtvarContext;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.painless.PainlessParser.ADD;
+import static org.elasticsearch.painless.PainlessParser.BWAND;
+import static org.elasticsearch.painless.PainlessParser.BWOR;
+import static org.elasticsearch.painless.PainlessParser.BWXOR;
+import static org.elasticsearch.painless.PainlessParser.DIV;
+import static org.elasticsearch.painless.PainlessParser.MUL;
+import static org.elasticsearch.painless.PainlessParser.REM;
+import static org.elasticsearch.painless.PainlessParser.SUB;
+
+class AnalyzerExternal {
+    private final Metadata metadata;
+    private final Definition definition;
+
+    private final Analyzer analyzer;
+    private final AnalyzerUtility utility;
+    private final AnalyzerCaster caster;
+    private final AnalyzerPromoter promoter;
+
+    AnalyzerExternal(final Metadata metadata, final Analyzer analyzer, final AnalyzerUtility utility,
+                     final AnalyzerCaster caster, final AnalyzerPromoter promoter) {
+        this.metadata = metadata;
+        this.definition = metadata.definition;
+
+        this.analyzer = analyzer;
+        this.utility = utility;
+        this.caster = caster;
+        this.promoter = promoter;
+    }
+
+    void processExtstart(final ExtstartContext ctx) {
+        final ExtprecContext precctx = ctx.extprec();
+        final ExtcastContext castctx = ctx.extcast();
+        final ExttypeContext typectx = ctx.exttype();
+        final ExtvarContext varctx = ctx.extvar();
+        final ExtnewContext newctx = ctx.extnew();
+        final ExtstringContext stringctx = ctx.extstring();
+
+        if (precctx != null) {
+            metadata.createExtNodeMetadata(ctx, precctx);
+            analyzer.visit(precctx);
+        } else if (castctx != null) {
+            metadata.createExtNodeMetadata(ctx, castctx);
+            analyzer.visit(castctx);
+        } else if (typectx != null) {
+            metadata.createExtNodeMetadata(ctx, typectx);
+            analyzer.visit(typectx);
+        } else if (varctx != null) {
+            metadata.createExtNodeMetadata(ctx, varctx);
+            analyzer.visit(varctx);
+        } else if (newctx != null) {
+            metadata.createExtNodeMetadata(ctx, newctx);
+            analyzer.visit(newctx);
+        } else if (stringctx != null) {
+            metadata.createExtNodeMetadata(ctx, stringctx);
+            analyzer.visit(stringctx);
+        } else {
+            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+        }
+    }
+
+    void processExtprec(final ExtprecContext ctx) {
+        final ExtNodeMetadata precenmd = metadata.getExtNodeMetadata(ctx);
+        final ParserRuleContext parent = precenmd.parent;
+        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
+
+        final ExtprecContext precctx = ctx.extprec();
+        final ExtcastContext castctx = ctx.extcast();
+        final ExttypeContext typectx = ctx.exttype();
+        final ExtvarContext varctx = ctx.extvar();
+        final ExtnewContext newctx = ctx.extnew();
+        final ExtstringContext stringctx = ctx.extstring();
+
+        final ExtdotContext dotctx = ctx.extdot();
+        final ExtbraceContext bracectx = ctx.extbrace();
+
+        if (dotctx != null || bracectx != null) {
+            ++parentemd.scope;
+        }
+
+        if (precctx != null) {
+            metadata.createExtNodeMetadata(parent, precctx);
+            analyzer.visit(precctx);
+        } else if (castctx != null) {
+            metadata.createExtNodeMetadata(parent, castctx);
+            analyzer.visit(castctx);
+        } else if (typectx != null) {
+            metadata.createExtNodeMetadata(parent, typectx);
+            analyzer.visit(typectx);
+        } else if (varctx != null) {
+            metadata.createExtNodeMetadata(parent, varctx);
+            analyzer.visit(varctx);
+        } else if (newctx != null) {
+            metadata.createExtNodeMetadata(parent, newctx);
+            analyzer.visit(newctx);
+        } else if (stringctx != null) {
+            metadata.createExtNodeMetadata(ctx, stringctx);
+            analyzer.visit(stringctx);
+        } else {
+            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+        }
+
+        parentemd.statement = false;
+
+        if (dotctx != null) {
+            --parentemd.scope;
+
+            metadata.createExtNodeMetadata(parent, dotctx);
+            analyzer.visit(dotctx);
+        } else if (bracectx != null) {
+            --parentemd.scope;
+
+            metadata.createExtNodeMetadata(parent, bracectx);
+            analyzer.visit(bracectx);
+        }
+    }
+
+    void processExtcast(final ExtcastContext ctx) {
+        final ExtNodeMetadata castenmd = metadata.getExtNodeMetadata(ctx);
+        final ParserRuleContext parent = castenmd.parent;
+        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
+
+        final ExtprecContext precctx = ctx.extprec();
+        final ExtcastContext castctx = ctx.extcast();
+        final ExttypeContext typectx = ctx.exttype();
+        final ExtvarContext varctx = ctx.extvar();
+        final ExtnewContext newctx = ctx.extnew();
+        final ExtstringContext stringctx = ctx.extstring();
+
+        if (precctx != null) {
+            metadata.createExtNodeMetadata(parent, precctx);
+            analyzer.visit(precctx);
+        } else if (castctx != null) {
+            metadata.createExtNodeMetadata(parent, castctx);
+            analyzer.visit(castctx);
+        } else if (typectx != null) {
+            metadata.createExtNodeMetadata(parent, typectx);
+            analyzer.visit(typectx);
+        } else if (varctx != null) {
+            metadata.createExtNodeMetadata(parent, varctx);
+            analyzer.visit(varctx);
+        } else if (newctx != null) {
+            metadata.createExtNodeMetadata(parent, newctx);
+            analyzer.visit(newctx);
+        } else if (stringctx != null) {
+            metadata.createExtNodeMetadata(ctx, stringctx);
+            analyzer.visit(stringctx);
+        } else {
+            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+        }
+
+        final DecltypeContext declctx = ctx.decltype();
+        final ExpressionMetadata declemd = metadata.createExpressionMetadata(declctx);
+        analyzer.visit(declctx);
+
+        castenmd.castTo = caster.getLegalCast(ctx, parentemd.current, declemd.from, true);
+        castenmd.type = declemd.from;
+        parentemd.current = declemd.from;
+        parentemd.statement = false;
+    }
+
+    void processExtbrace(final ExtbraceContext ctx) {
+        final ExtNodeMetadata braceenmd = metadata.getExtNodeMetadata(ctx);
+        final ParserRuleContext parent = braceenmd.parent;
+        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
+
+        final boolean array = parentemd.current.sort == Sort.ARRAY;
+        final boolean def = parentemd.current.sort == Sort.DEF;
+        boolean map = false;
+        boolean list = false;
+
+        try {
+            parentemd.current.clazz.asSubclass(Map.class);
+            map = true;
+        } catch (final ClassCastException exception) {
+            // Do nothing.
+        }
+
+        try {
+            parentemd.current.clazz.asSubclass(List.class);
+            list = true;
+        } catch (final ClassCastException exception) {
+            // Do nothing.
+        }
+
+        final ExtdotContext dotctx = ctx.extdot();
+        final ExtbraceContext bracectx = ctx.extbrace();
+
+        braceenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null;
+
+        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
+        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
+
+        if (array || def) {
+            expremd.to = array ? definition.intType : definition.objectType;
+            analyzer.visit(exprctx);
+            caster.markCast(expremd);
+
+            braceenmd.target = "#brace";
+            braceenmd.type = def ? definition.defType :
+                definition.getType(parentemd.current.struct, parentemd.current.type.getDimensions() - 1);
+            analyzeLoadStoreExternal(ctx);
+            parentemd.current = braceenmd.type;
+
+            if (dotctx != null) {
+                metadata.createExtNodeMetadata(parent, dotctx);
+                analyzer.visit(dotctx);
+            } else if (bracectx != null) {
+                metadata.createExtNodeMetadata(parent, bracectx);
+                analyzer.visit(bracectx);
+            }
+        } else {
+            final boolean store = braceenmd.last && parentemd.storeExpr != null;
+            final boolean get = parentemd.read || parentemd.token > 0 || !braceenmd.last;
+            final boolean set = braceenmd.last && store;
+
+            Method getter;
+            Method setter;
+            Type valuetype;
+            Type settype;
+
+            if (map) {
+                getter = parentemd.current.struct.methods.get("get");
+                setter = parentemd.current.struct.methods.get("put");
+
+                if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1)) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
+                        "Illegal map get shortcut for type [" + parentemd.current.name + "].");
+                }
+
+                if (setter != null && setter.arguments.size() != 2) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
+                        "Illegal map set shortcut for type [" + parentemd.current.name + "].");
+                }
+
+                if (getter != null && setter != null && (!getter.arguments.get(0).equals(setter.arguments.get(0))
+                    || !getter.rtn.equals(setter.arguments.get(1)))) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Shortcut argument types must match.");
+                }
+
+                valuetype = setter != null ? setter.arguments.get(0) : getter != null ? getter.arguments.get(0) : null;
+                settype = setter == null ? null : setter.arguments.get(1);
+            } else if (list) {
+                getter = parentemd.current.struct.methods.get("get");
+                setter = parentemd.current.struct.methods.get("set");
+
+                if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1 ||
+                    getter.arguments.get(0).sort != Sort.INT)) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
+                        "Illegal list get shortcut for type [" + parentemd.current.name + "].");
+                }
+
+                if (setter != null && (setter.arguments.size() != 2 || setter.arguments.get(0).sort != Sort.INT)) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
+                        "Illegal list set shortcut for type [" + parentemd.current.name + "].");
+                }
+
+                if (getter != null && setter != null && (!getter.arguments.get(0).equals(setter.arguments.get(0))
+                    || !getter.rtn.equals(setter.arguments.get(1)))) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Shortcut argument types must match.");
+                }
+
+                valuetype = definition.intType;
+                settype = setter == null ? null : setter.arguments.get(1);
+            } else {
+                throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+            }
+
+            if ((get || set) && (!get || getter != null) && (!set || setter != null)) {
+                expremd.to = valuetype;
+                analyzer.visit(exprctx);
+                caster.markCast(expremd);
+
+                braceenmd.target = new Object[] {getter, setter, true, null};
+                braceenmd.type = get ? getter.rtn : settype;
+                analyzeLoadStoreExternal(ctx);
+                parentemd.current = get ? getter.rtn : setter.rtn;
+            }
+        }
+
+        if (braceenmd.target == null) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
+                "Attempting to address a non-array type [" + parentemd.current.name + "] as an array.");
+        }
+    }
+
+    void processExtdot(final ExtdotContext ctx) {
+        final ExtNodeMetadata dotemnd = metadata.getExtNodeMetadata(ctx);
+        final ParserRuleContext parent = dotemnd.parent;
+
+        final ExtcallContext callctx = ctx.extcall();
+        final ExtfieldContext fieldctx = ctx.extfield();
+
+        if (callctx != null) {
+            metadata.createExtNodeMetadata(parent, callctx);
+            analyzer.visit(callctx);
+        } else if (fieldctx != null) {
+            metadata.createExtNodeMetadata(parent, fieldctx);
+            analyzer.visit(fieldctx);
+        }
+    }
+
+    void processExttype(final ExttypeContext ctx) {
+        final ExtNodeMetadata typeenmd = metadata.getExtNodeMetadata(ctx);
+        final ParserRuleContext parent = typeenmd.parent;
+        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
+
+        if (parentemd.current != null) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unexpected static type.");
+        }
+
+        final String typestr = ctx.TYPE().getText();
+        typeenmd.type = definition.getType(typestr);
+        parentemd.current = typeenmd.type;
+        parentemd.statik = true;
+
+        final ExtdotContext dotctx = ctx.extdot();
+        metadata.createExtNodeMetadata(parent, dotctx);
+        analyzer.visit(dotctx);
+    }
+
+    void processExtcall(final ExtcallContext ctx) {
+        final ExtNodeMetadata callenmd = metadata.getExtNodeMetadata(ctx);
+        final ParserRuleContext parent = callenmd.parent;
+        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
+
+        final ExtdotContext dotctx = ctx.extdot();
+        final ExtbraceContext bracectx = ctx.extbrace();
+
+        callenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null;
+
+        final String name = ctx.EXTID().getText();
+
+        if (parentemd.current.sort == Sort.ARRAY) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unexpected call [" + name + "] on an array.");
+        } else if (callenmd.last && parentemd.storeExpr != null) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Cannot assign a value to a call [" + name + "].");
+        }
+
+        final Struct struct = parentemd.current.struct;
+        final List<ExpressionContext> arguments = ctx.arguments().expression();
+        final int size = arguments.size();
+        Type[] types;
+
+        final Method method = parentemd.statik ? struct.functions.get(name) : struct.methods.get(name);
+        final boolean def = parentemd.current.sort == Sort.DEF;
+
+        if (method == null && !def) {
+            throw new IllegalArgumentException(
+                AnalyzerUtility.error(ctx) + "Unknown call [" + name + "] on type [" + struct.name + "].");
+        } else if (method != null) {
+            types = new Type[method.arguments.size()];
+            method.arguments.toArray(types);
+
+            callenmd.target = method;
+            callenmd.type = method.rtn;
+            parentemd.statement = !parentemd.read && callenmd.last;
+            parentemd.current = method.rtn;
+
+            if (size != types.length) {
+                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "When calling [" + name + "] on type " +
+                    "[" + struct.name + "] expected [" + types.length + "] arguments," +
+                    " but found [" + arguments.size() + "].");
+            }
+        } else {
+            types = new Type[arguments.size()];
+            Arrays.fill(types, definition.defType);
+
+            callenmd.target = name;
+            callenmd.type = definition.defType;
+            parentemd.statement = !parentemd.read && callenmd.last;
+            parentemd.current = callenmd.type;
+        }
+
+        for (int argument = 0; argument < size; ++argument) {
+            final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(arguments.get(argument));
+            final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
+            expremd.to = types[argument];
+            analyzer.visit(exprctx);
+            caster.markCast(expremd);
+        }
+
+        parentemd.statik = false;
+
+        if (dotctx != null) {
+            metadata.createExtNodeMetadata(parent, dotctx);
+            analyzer.visit(dotctx);
+        } else if (bracectx != null) {
+            metadata.createExtNodeMetadata(parent, bracectx);
+            analyzer.visit(bracectx);
+        }
+    }
+
+    void processExtvar(final ExtvarContext ctx) {
+        final ExtNodeMetadata varenmd = metadata.getExtNodeMetadata(ctx);
+        final ParserRuleContext parent = varenmd.parent;
+        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
+
+        final String name = ctx.ID().getText();
+
+        final ExtdotContext dotctx = ctx.extdot();
+        final ExtbraceContext bracectx = ctx.extbrace();
+
+        if (parentemd.current != null) {
+            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected variable [" + name + "] load.");
+        }
+
+        varenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null;
+
+        final Variable variable = utility.getVariable(name);
+
+        if (variable == null) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unknown variable [" + name + "].");
+        }
+
+        varenmd.target = variable.slot;
+        varenmd.type = variable.type;
+        analyzeLoadStoreExternal(ctx);
+        parentemd.current = varenmd.type;
+
+        if (dotctx != null) {
+            metadata.createExtNodeMetadata(parent, dotctx);
+            analyzer.visit(dotctx);
+        } else if (bracectx != null) {
+            metadata.createExtNodeMetadata(parent, bracectx);
+            analyzer.visit(bracectx);
+        }
+    }
+
+    void processExtfield(final ExtfieldContext ctx) {
+        final ExtNodeMetadata memberenmd = metadata.getExtNodeMetadata(ctx);
+        final ParserRuleContext parent = memberenmd.parent;
+        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
+
+        if (ctx.EXTID() == null && ctx.EXTINTEGER() == null) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+        }
+
+        final String value = ctx.EXTID() == null ? ctx.EXTINTEGER().getText() : ctx.EXTID().getText();
+
+        final ExtdotContext dotctx = ctx.extdot();
+        final ExtbraceContext bracectx = ctx.extbrace();
+
+        memberenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null;
+        final boolean store = memberenmd.last && parentemd.storeExpr != null;
+
+        if (parentemd.current == null) {
+            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected field [" + value + "] load.");
+        }
+
+        if (parentemd.current.sort == Sort.ARRAY) {
+            if ("length".equals(value)) {
+                if (!parentemd.read) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Must read array field [length].");
+                } else if (store) {
+                    throw new IllegalArgumentException(
+                        AnalyzerUtility.error(ctx) + "Cannot write to read-only array field [length].");
+                }
+
+                memberenmd.target = "#length";
+                memberenmd.type = definition.intType;
+                parentemd.current = definition.intType;
+            } else {
+                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unexpected array field [" + value + "].");
+            }
+        } else if (parentemd.current.sort == Sort.DEF) {
+            memberenmd.target = value;
+            memberenmd.type = definition.defType;
+            analyzeLoadStoreExternal(ctx);
+            parentemd.current = memberenmd.type;
+        } else {
+            final Struct struct = parentemd.current.struct;
+            final Field field = parentemd.statik ? struct.statics.get(value) : struct.members.get(value);
+
+            if (field != null) {
+                if (store && java.lang.reflect.Modifier.isFinal(field.reflect.getModifiers())) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Cannot write to read-only" +
+                        " field [" + value + "] for type [" + struct.name + "].");
+                }
+
+                memberenmd.target = field;
+                memberenmd.type = field.type;
+                analyzeLoadStoreExternal(ctx);
+                parentemd.current = memberenmd.type;
+            } else {
+                final boolean get = parentemd.read || parentemd.token > 0 || !memberenmd.last;
+                final boolean set = memberenmd.last && store;
+
+                Method getter = struct.methods.get("get" + Character.toUpperCase(value.charAt(0)) + value.substring(1));
+                Method setter = struct.methods.get("set" + Character.toUpperCase(value.charAt(0)) + value.substring(1));
+                Object constant = null;
+
+                if (getter != null && (getter.rtn.sort == Sort.VOID || !getter.arguments.isEmpty())) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
+                        "Illegal get shortcut on field [" + value + "] for type [" + struct.name + "].");
+                }
+
+                if (setter != null && (setter.rtn.sort != Sort.VOID || setter.arguments.size() != 1)) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
+                        "Illegal set shortcut on field [" + value + "] for type [" + struct.name + "].");
+                }
+
+                Type settype = setter == null ? null : setter.arguments.get(0);
+
+                if (getter == null && setter == null) {
+                    if (ctx.EXTID() != null) {
+                        try {
+                            parentemd.current.clazz.asSubclass(Map.class);
+
+                            getter = parentemd.current.struct.methods.get("get");
+                            setter = parentemd.current.struct.methods.get("put");
+
+                            if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1 ||
+                                getter.arguments.get(0).sort != Sort.STRING)) {
+                                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
+                                    "Illegal map get shortcut [" + value + "] for type [" + struct.name + "].");
+                            }
+
+                            if (setter != null && (setter.arguments.size() != 2 ||
+                                setter.arguments.get(0).sort != Sort.STRING)) {
+                                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
+                                    "Illegal map set shortcut [" + value + "] for type [" + struct.name + "].");
+                            }
+
+                            if (getter != null && setter != null && !getter.rtn.equals(setter.arguments.get(1))) {
+                                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Shortcut argument types must match.");
+                            }
+
+                            settype = setter == null ? null : setter.arguments.get(1);
+                            constant = value;
+                        } catch (ClassCastException exception) {
+                            //Do nothing.
+                        }
+                    } else if (ctx.EXTINTEGER() != null) {
+                        try {
+                            parentemd.current.clazz.asSubclass(List.class);
+
+                            getter = parentemd.current.struct.methods.get("get");
+                            setter = parentemd.current.struct.methods.get("set");
+
+                            if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1 ||
+                                getter.arguments.get(0).sort != Sort.INT)) {
+                                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
+                                    "Illegal list get shortcut [" + value + "] for type [" + struct.name + "].");
+                            }
+
+                            if (setter != null && (setter.rtn.sort != Sort.VOID || setter.arguments.size() != 2 ||
+                                setter.arguments.get(0).sort != Sort.INT)) {
+                                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
+                                    "Illegal list set shortcut [" + value + "] for type [" + struct.name + "].");
+                            }
+
+                            if (getter != null && setter != null && !getter.rtn.equals(setter.arguments.get(1))) {
+                                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Shortcut argument types must match.");
+                            }
+
+                            settype = setter == null ? null : setter.arguments.get(1);
+
+                            try {
+                                constant = Integer.parseInt(value);
+                            } catch (NumberFormatException exception) {
+                                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
+                                    "Illegal list shortcut value [" + value + "].");
+                            }
+                        } catch (ClassCastException exception) {
+                            //Do nothing.
+                        }
+                    } else {
+                        throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+                    }
+                }
+
+                if ((get || set) && (!get || getter != null) && (!set || setter != null)) {
+                    memberenmd.target = new Object[] {getter, setter, constant != null, constant};
+                    memberenmd.type = get ? getter.rtn : settype;
+                    analyzeLoadStoreExternal(ctx);
+                    parentemd.current = get ? getter.rtn : setter.rtn;
+                }
+            }
+
+            if (memberenmd.target == null) {
+                throw new IllegalArgumentException(
+                    AnalyzerUtility.error(ctx) + "Unknown field [" + value + "] for type [" + struct.name + "].");
+            }
+        }
+
+        parentemd.statik = false;
+
+        if (dotctx != null) {
+            metadata.createExtNodeMetadata(parent, dotctx);
+            analyzer.visit(dotctx);
+        } else if (bracectx != null) {
+            metadata.createExtNodeMetadata(parent, bracectx);
+            analyzer.visit(bracectx);
+        }
+    }
+
+    void processExtnew(final ExtnewContext ctx) {
+        final ExtNodeMetadata newenmd = metadata.getExtNodeMetadata(ctx);
+        final ParserRuleContext parent = newenmd.parent;
+        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
+
+        final ExtdotContext dotctx = ctx.extdot();
+        final ExtbraceContext bracectx = ctx.extbrace();
+
+        newenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null;
+
+        final String name = ctx.TYPE().getText();
+        final Struct struct = definition.structs.get(name);
+
+        if (parentemd.current != null) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unexpected new call.");
+        } else if (struct == null) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Specified type [" + name + "] not found.");
+        } else if (newenmd.last && parentemd.storeExpr != null) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Cannot assign a value to a new call.");
+        }
+
+        final boolean newclass = ctx.arguments() != null;
+        final boolean newarray = !ctx.expression().isEmpty();
+
+        final List<ExpressionContext> arguments = newclass ? ctx.arguments().expression() : ctx.expression();
+        final int size = arguments.size();
+
+        Type[] types;
+
+        if (newarray) {
+            if (!parentemd.read) {
+                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "A newly created array must be assigned.");
+            }
+
+            types = new Type[size];
+            Arrays.fill(types, definition.intType);
+
+            newenmd.target = "#makearray";
+
+            if (size > 1) {
+                newenmd.type = definition.getType(struct, size);
+                parentemd.current = newenmd.type;
+            } else if (size == 1) {
+                newenmd.type = definition.getType(struct, 0);
+                parentemd.current = definition.getType(struct, 1);
+            } else {
+                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "A newly created array cannot have zero dimensions.");
+            }
+        } else if (newclass) {
+            final Constructor constructor = struct.constructors.get("new");
+
+            if (constructor != null) {
+                types = new Type[constructor.arguments.size()];
+                constructor.arguments.toArray(types);
+
+                newenmd.target = constructor;
+                newenmd.type = definition.getType(struct, 0);
+                parentemd.statement = !parentemd.read && newenmd.last;
+                parentemd.current = newenmd.type;
+            } else {
+                throw new IllegalArgumentException(
+                    AnalyzerUtility.error(ctx) + "Unknown new call on type [" + struct.name + "].");
+            }
+        } else {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unknown state.");
+        }
+
+        if (size != types.length) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "When calling [" + name + "] on type " +
+                "[" + struct.name + "] expected [" + types.length + "] arguments," +
+                " but found [" + arguments.size() + "].");
+        }
+
+        for (int argument = 0; argument < size; ++argument) {
+            final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(arguments.get(argument));
+            final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
+            expremd.to = types[argument];
+            analyzer.visit(exprctx);
+            caster.markCast(expremd);
+        }
+
+        if (dotctx != null) {
+            metadata.createExtNodeMetadata(parent, dotctx);
+            analyzer.visit(dotctx);
+        } else if (bracectx != null) {
+            metadata.createExtNodeMetadata(parent, bracectx);
+            analyzer.visit(bracectx);
+        }
+    }
+
+    void processExtstring(final ExtstringContext ctx) {
+        final ExtNodeMetadata memberenmd = metadata.getExtNodeMetadata(ctx);
+        final ParserRuleContext parent = memberenmd.parent;
+        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
+
+        final String string = ctx.STRING().getText();
+
+        final ExtdotContext dotctx = ctx.extdot();
+        final ExtbraceContext bracectx = ctx.extbrace();
+
+        memberenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null;
+        final boolean store = memberenmd.last && parentemd.storeExpr != null;
+
+        if (parentemd.current != null) {
+            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected String constant [" + string + "].");
+        }
+
+        if (!parentemd.read) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Must read String constant [" + string + "].");
+        } else if (store) {
+            throw new IllegalArgumentException(
+                AnalyzerUtility.error(ctx) + "Cannot write to read-only String constant [" + string + "].");
+        }
+
+        memberenmd.target = string;
+        memberenmd.type = definition.stringType;
+        parentemd.current = definition.stringType;
+
+        if (memberenmd.last) {
+            parentemd.constant = string;
+        }
+
+        if (dotctx != null) {
+            metadata.createExtNodeMetadata(parent, dotctx);
+            analyzer.visit(dotctx);
+        } else if (bracectx != null) {
+            metadata.createExtNodeMetadata(parent, bracectx);
+            analyzer.visit(bracectx);
+        }
+    }
+
+    private void analyzeLoadStoreExternal(final ParserRuleContext source) {
+        final ExtNodeMetadata extenmd = metadata.getExtNodeMetadata(source);
+        final ParserRuleContext parent = extenmd.parent;
+        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
+
+        if (extenmd.last && parentemd.storeExpr != null) {
+            final ParserRuleContext store = parentemd.storeExpr;
+            final ExpressionMetadata storeemd = metadata.createExpressionMetadata(parentemd.storeExpr);
+            final int token = parentemd.token;
+
+            if (token > 0) {
+                analyzer.visit(store);
+
+                final boolean add = token == ADD;
+                final boolean xor = token == BWAND || token == BWXOR || token == BWOR;
+                final boolean decimal = token == MUL || token == DIV || token == REM || token == SUB;
+
+                extenmd.promote = add ? promoter.promoteAdd(extenmd.type, storeemd.from) :
+                    xor ? promoter.promoteXor(extenmd.type, storeemd.from) :
+                        promoter.promoteNumeric(extenmd.type, storeemd.from, decimal, true);
+
+                if (extenmd.promote == null) {
+                    throw new IllegalArgumentException("Cannot apply compound assignment to " +
+                        "types [" + extenmd.type.name + "] and [" + storeemd.from.name + "].");
+                }
+
+                extenmd.castFrom = caster.getLegalCast(source, extenmd.type, extenmd.promote, false);
+                extenmd.castTo = caster.getLegalCast(source, extenmd.promote, extenmd.type, true);
+
+                storeemd.to = add && extenmd.promote.sort == Sort.STRING ? storeemd.from : extenmd.promote;
+                caster.markCast(storeemd);
+            } else {
+                storeemd.to = extenmd.type;
+                analyzer.visit(store);
+                caster.markCast(storeemd);
+            }
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerPromoter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerPromoter.java
new file mode 100644
index 00000000000..ff77fb06d93
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerPromoter.java
@@ -0,0 +1,281 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.painless.Definition.Pair;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Definition.Type;
+
+class AnalyzerPromoter {
+    private final Definition definition;
+
+    AnalyzerPromoter(final Definition definition) {
+        this.definition = definition;
+    }
+
+    Type promoteNumeric(final Type from, final boolean decimal, final boolean primitive) {
+        final Sort sort = from.sort;
+
+        if (sort == Sort.DEF) {
+            return definition.defType;
+        } else if ((sort == Sort.DOUBLE || sort == Sort.DOUBLE_OBJ || sort == Sort.NUMBER) && decimal) {
+            return primitive ? definition.doubleType : definition.doubleobjType;
+        } else if ((sort == Sort.FLOAT || sort == Sort.FLOAT_OBJ) && decimal) {
+            return primitive ? definition.floatType : definition.floatobjType;
+        } else if (sort == Sort.LONG || sort == Sort.LONG_OBJ || sort == Sort.NUMBER) {
+            return primitive ? definition.longType : definition.longobjType;
+        } else if (sort.numeric) {
+            return primitive ? definition.intType : definition.intobjType;
+        }
+
+        return null;
+    }
+
+    Type promoteNumeric(final Type from0, final Type from1, final boolean decimal, final boolean primitive) {
+        final Sort sort0 = from0.sort;
+        final Sort sort1 = from1.sort;
+
+        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
+            return definition.defType;
+        }
+
+        if (decimal) {
+            if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ || sort0 == Sort.NUMBER ||
+                sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ || sort1 == Sort.NUMBER) {
+                return primitive ? definition.doubleType : definition.doubleobjType;
+            } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) {
+                return primitive ? definition.floatType : definition.floatobjType;
+            }
+        }
+
+        if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ || sort0 == Sort.NUMBER ||
+            sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ || sort1 == Sort.NUMBER) {
+            return primitive ? definition.longType : definition.longobjType;
+        } else if (sort0.numeric && sort1.numeric) {
+            return primitive ? definition.intType : definition.intobjType;
+        }
+
+        return null;
+    }
+
+    Type promoteAdd(final Type from0, final Type from1) {
+        final Sort sort0 = from0.sort;
+        final Sort sort1 = from1.sort;
+
+        if (sort0 == Sort.STRING || sort1 == Sort.STRING) {
+            return definition.stringType;
+        }
+
+        return promoteNumeric(from0, from1, true, true);
+    }
+
+    Type promoteXor(final Type from0, final Type from1) {
+        final Sort sort0 = from0.sort;
+        final Sort sort1 = from1.sort;
+
+        if (sort0.bool || sort1.bool) {
+            return definition.booleanType;
+        }
+
+        return promoteNumeric(from0, from1, false, true);
+    }
+
+    Type promoteEquality(final Type from0, final Type from1) {
+        final Sort sort0 = from0.sort;
+        final Sort sort1 = from1.sort;
+
+        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
+            return definition.defType;
+        }
+
+        final boolean primitive = sort0.primitive && sort1.primitive;
+
+        if (sort0.bool && sort1.bool) {
+            return primitive ? definition.booleanType : definition.booleanobjType;
+        }
+
+        if (sort0.numeric && sort1.numeric) {
+            return promoteNumeric(from0, from1, true, primitive);
+        }
+
+        return definition.objectType;
+    }
+
+    Type promoteReference(final Type from0, final Type from1) {
+        final Sort sort0 = from0.sort;
+        final Sort sort1 = from1.sort;
+
+        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
+            return definition.defType;
+        }
+
+        if (sort0.primitive && sort1.primitive) {
+            if (sort0.bool && sort1.bool) {
+                return definition.booleanType;
+            }
+
+            if (sort0.numeric && sort1.numeric) {
+                return promoteNumeric(from0, from1, true, true);
+            }
+        }
+
+        return definition.objectType;
+    }
+
+    Type promoteConditional(final Type from0, final Type from1, final Object const0, final Object const1) {
+        if (from0.equals(from1)) {
+            return from0;
+        }
+
+        final Sort sort0 = from0.sort;
+        final Sort sort1 = from1.sort;
+
+        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
+            return definition.defType;
+        }
+
+        final boolean primitive = sort0.primitive && sort1.primitive;
+
+        if (sort0.bool && sort1.bool) {
+            return primitive ? definition.booleanType : definition.booleanobjType;
+        }
+
+        if (sort0.numeric && sort1.numeric) {
+            if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ || sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ) {
+                return primitive ? definition.doubleType : definition.doubleobjType;
+            } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) {
+                return primitive ? definition.floatType : definition.floatobjType;
+            } else if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ || sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ) {
+                return sort0.primitive && sort1.primitive ? definition.longType : definition.longobjType;
+            } else {
+                if (sort0 == Sort.BYTE || sort0 == Sort.BYTE_OBJ) {
+                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
+                        return primitive ? definition.byteType : definition.byteobjType;
+                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
+                        if (const1 != null) {
+                            final short constant = (short)const1;
+
+                            if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
+                                return primitive ? definition.byteType : definition.byteobjType;
+                            }
+                        }
+
+                        return primitive ? definition.shortType : definition.shortobjType;
+                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
+                        return primitive ? definition.intType : definition.intobjType;
+                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
+                        if (const1 != null) {
+                            final int constant = (int)const1;
+
+                            if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
+                                return primitive ? definition.byteType : definition.byteobjType;
+                            }
+                        }
+
+                        return primitive ? definition.intType : definition.intobjType;
+                    }
+                } else if (sort0 == Sort.SHORT || sort0 == Sort.SHORT_OBJ) {
+                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
+                        if (const0 != null) {
+                            final short constant = (short)const0;
+
+                            if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
+                                return primitive ? definition.byteType : definition.byteobjType;
+                            }
+                        }
+
+                        return primitive ? definition.shortType : definition.shortobjType;
+                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
+                        return primitive ? definition.shortType : definition.shortobjType;
+                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
+                        return primitive ? definition.intType : definition.intobjType;
+                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
+                        if (const1 != null) {
+                            final int constant = (int)const1;
+
+                            if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
+                                return primitive ? definition.shortType : definition.shortobjType;
+                            }
+                        }
+
+                        return primitive ? definition.intType : definition.intobjType;
+                    }
+                } else if (sort0 == Sort.CHAR || sort0 == Sort.CHAR_OBJ) {
+                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
+                        return primitive ? definition.intType : definition.intobjType;
+                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
+                        return primitive ? definition.intType : definition.intobjType;
+                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
+                        return primitive ? definition.charType : definition.charobjType;
+                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
+                        if (const1 != null) {
+                            final int constant = (int)const1;
+
+                            if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
+                                return primitive ? definition.byteType : definition.byteobjType;
+                            }
+                        }
+
+                        return primitive ? definition.intType : definition.intobjType;
+                    }
+                } else if (sort0 == Sort.INT || sort0 == Sort.INT_OBJ) {
+                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
+                        if (const0 != null) {
+                            final int constant = (int)const0;
+
+                            if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
+                                return primitive ? definition.byteType : definition.byteobjType;
+                            }
+                        }
+
+                        return primitive ? definition.intType : definition.intobjType;
+                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
+                        if (const0 != null) {
+                            final int constant = (int)const0;
+
+                            if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
+                                return primitive ? definition.byteType : definition.byteobjType;
+                            }
+                        }
+
+                        return primitive ? definition.intType : definition.intobjType;
+                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
+                        if (const0 != null) {
+                            final int constant = (int)const0;
+
+                            if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
+                                return primitive ? definition.byteType : definition.byteobjType;
+                            }
+                        }
+
+                        return primitive ? definition.intType : definition.intobjType;
+                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
+                        return primitive ? definition.intType : definition.intobjType;
+                    }
+                }
+            }
+        }
+
+        final Pair pair = new Pair(from0, from1);
+        final Type bound = definition.bounds.get(pair);
+
+        return bound == null ? definition.objectType : bound;
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerStatement.java
new file mode 100644
index 00000000000..e44336035e6
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerStatement.java
@@ -0,0 +1,581 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Metadata.ExpressionMetadata;
+import org.elasticsearch.painless.Metadata.StatementMetadata;
+import org.elasticsearch.painless.PainlessParser.AfterthoughtContext;
+import org.elasticsearch.painless.PainlessParser.BlockContext;
+import org.elasticsearch.painless.PainlessParser.BreakContext;
+import org.elasticsearch.painless.PainlessParser.ContinueContext;
+import org.elasticsearch.painless.PainlessParser.DeclContext;
+import org.elasticsearch.painless.PainlessParser.DeclarationContext;
+import org.elasticsearch.painless.PainlessParser.DecltypeContext;
+import org.elasticsearch.painless.PainlessParser.DeclvarContext;
+import org.elasticsearch.painless.PainlessParser.DoContext;
+import org.elasticsearch.painless.PainlessParser.ExprContext;
+import org.elasticsearch.painless.PainlessParser.ExpressionContext;
+import org.elasticsearch.painless.PainlessParser.ForContext;
+import org.elasticsearch.painless.PainlessParser.IfContext;
+import org.elasticsearch.painless.PainlessParser.InitializerContext;
+import org.elasticsearch.painless.PainlessParser.MultipleContext;
+import org.elasticsearch.painless.PainlessParser.ReturnContext;
+import org.elasticsearch.painless.PainlessParser.SingleContext;
+import org.elasticsearch.painless.PainlessParser.SourceContext;
+import org.elasticsearch.painless.PainlessParser.StatementContext;
+import org.elasticsearch.painless.PainlessParser.ThrowContext;
+import org.elasticsearch.painless.PainlessParser.TrapContext;
+import org.elasticsearch.painless.PainlessParser.TryContext;
+import org.elasticsearch.painless.PainlessParser.WhileContext;
+
+import java.util.List;
+
+class AnalyzerStatement {
+    private final Metadata metadata;
+    private final Definition definition;
+
+    private final Analyzer analyzer;
+    private final AnalyzerUtility utility;
+    private final AnalyzerCaster caster;
+
+    AnalyzerStatement(final Metadata metadata, final Analyzer analyzer,
+                      final AnalyzerUtility utility, final AnalyzerCaster caster) {
+        this.metadata = metadata;
+        this.definition = metadata.definition;
+
+        this.analyzer = analyzer;
+        this.utility = utility;
+        this.caster = caster;
+    }
+
+    void processSource(final SourceContext ctx) {
+        final StatementMetadata sourcesmd = metadata.getStatementMetadata(ctx);
+        final List<StatementContext> statectxs = ctx.statement();
+        final StatementContext lastctx = statectxs.get(statectxs.size() - 1);
+
+        utility.incrementScope();
+
+        for (final StatementContext statectx : statectxs) {
+            if (sourcesmd.allLast) {
+                throw new IllegalArgumentException(AnalyzerUtility.error(statectx) +
+                    "Statement will never be executed because all prior paths escape.");
+            }
+
+            final StatementMetadata statesmd = metadata.createStatementMetadata(statectx);
+            statesmd.lastSource = statectx == lastctx;
+            analyzer.visit(statectx);
+
+            sourcesmd.methodEscape = statesmd.methodEscape;
+            sourcesmd.allLast = statesmd.allLast;
+        }
+
+        utility.decrementScope();
+    }
+
+    void processIf(final IfContext ctx) {
+        final StatementMetadata ifsmd = metadata.getStatementMetadata(ctx);
+
+        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
+        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
+        expremd.to = definition.booleanType;
+        analyzer.visit(exprctx);
+        caster.markCast(expremd);
+
+        if (expremd.postConst != null) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "If statement is not necessary.");
+        }
+
+        final BlockContext blockctx0 = ctx.block(0);
+        final StatementMetadata blocksmd0 = metadata.createStatementMetadata(blockctx0);
+        blocksmd0.lastSource = ifsmd.lastSource;
+        blocksmd0.inLoop = ifsmd.inLoop;
+        blocksmd0.lastLoop = ifsmd.lastLoop;
+        utility.incrementScope();
+        analyzer.visit(blockctx0);
+        utility.decrementScope();
+
+        ifsmd.anyContinue = blocksmd0.anyContinue;
+        ifsmd.anyBreak = blocksmd0.anyBreak;
+
+        ifsmd.count = blocksmd0.count;
+
+        if (ctx.ELSE() != null) {
+            final BlockContext blockctx1 = ctx.block(1);
+            final StatementMetadata blocksmd1 = metadata.createStatementMetadata(blockctx1);
+            blocksmd1.lastSource = ifsmd.lastSource;
+            utility.incrementScope();
+            analyzer.visit(blockctx1);
+            utility.decrementScope();
+
+            ifsmd.methodEscape = blocksmd0.methodEscape && blocksmd1.methodEscape;
+            ifsmd.loopEscape = blocksmd0.loopEscape && blocksmd1.loopEscape;
+            ifsmd.allLast = blocksmd0.allLast && blocksmd1.allLast;
+            ifsmd.anyContinue |= blocksmd1.anyContinue;
+            ifsmd.anyBreak |= blocksmd1.anyBreak;
+
+            ifsmd.count = Math.max(ifsmd.count, blocksmd1.count);
+        }
+    }
+
+    void processWhile(final WhileContext ctx) {
+        final StatementMetadata whilesmd = metadata.getStatementMetadata(ctx);
+
+        utility.incrementScope();
+
+        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
+        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
+        expremd.to = definition.booleanType;
+        analyzer.visit(exprctx);
+        caster.markCast(expremd);
+
+        boolean continuous = false;
+
+        if (expremd.postConst != null) {
+            continuous = (boolean)expremd.postConst;
+
+            if (!continuous) {
+                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "The loop will never be executed.");
+            }
+
+            if (ctx.empty() != null) {
+                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "The loop will never exit.");
+            }
+        }
+
+        final BlockContext blockctx = ctx.block();
+
+        if (blockctx != null) {
+            final StatementMetadata blocksmd = metadata.createStatementMetadata(blockctx);
+            blocksmd.beginLoop = true;
+            blocksmd.inLoop = true;
+            analyzer.visit(blockctx);
+
+            if (blocksmd.loopEscape && !blocksmd.anyContinue) {
+                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "All paths escape so the loop is not necessary.");
+            }
+
+            if (continuous && !blocksmd.anyBreak) {
+                whilesmd.methodEscape = true;
+                whilesmd.allLast = true;
+            }
+        }
+
+        whilesmd.count = 1;
+
+        utility.decrementScope();
+    }
+
+    void processDo(final DoContext ctx) {
+        final StatementMetadata dosmd = metadata.getStatementMetadata(ctx);
+
+        utility.incrementScope();
+
+        final BlockContext blockctx = ctx.block();
+        final StatementMetadata blocksmd = metadata.createStatementMetadata(blockctx);
+        blocksmd.beginLoop = true;
+        blocksmd.inLoop = true;
+        analyzer.visit(blockctx);
+
+        if (blocksmd.loopEscape && !blocksmd.anyContinue) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "All paths escape so the loop is not necessary.");
+        }
+
+        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
+        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
+        expremd.to = definition.booleanType;
+        analyzer.visit(exprctx);
+        caster.markCast(expremd);
+
+        if (expremd.postConst != null) {
+            final boolean continuous = (boolean)expremd.postConst;
+
+            if (!continuous) {
+                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "All paths escape so the loop is not necessary.");
+            }
+
+            if (!blocksmd.anyBreak) {
+                dosmd.methodEscape = true;
+                dosmd.allLast = true;
+            }
+        }
+
+        dosmd.count = 1;
+
+        utility.decrementScope();
+    }
+
+    void processFor(final ForContext ctx) {
+        final StatementMetadata forsmd = metadata.getStatementMetadata(ctx);
+        boolean continuous = false;
+
+        utility.incrementScope();
+
+        final InitializerContext initctx = ctx.initializer();
+
+        if (initctx != null) {
+            metadata.createStatementMetadata(initctx);
+            analyzer.visit(initctx);
+        }
+
+        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
+
+        if (exprctx != null) {
+            final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
+            expremd.to = definition.booleanType;
+            analyzer.visit(exprctx);
+            caster.markCast(expremd);
+
+            if (expremd.postConst != null) {
+                continuous = (boolean)expremd.postConst;
+
+                if (!continuous) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "The loop will never be executed.");
+                }
+
+                if (ctx.empty() != null) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "The loop is continuous.");
+                }
+            }
+        } else {
+            continuous = true;
+        }
+
+        final AfterthoughtContext atctx = ctx.afterthought();
+
+        if (atctx != null) {
+            metadata.createStatementMetadata(atctx);
+            analyzer.visit(atctx);
+        }
+
+        final BlockContext blockctx = ctx.block();
+
+        if (blockctx != null) {
+            final StatementMetadata blocksmd = metadata.createStatementMetadata(blockctx);
+            blocksmd.beginLoop = true;
+            blocksmd.inLoop = true;
+            analyzer.visit(blockctx);
+
+            if (blocksmd.loopEscape && !blocksmd.anyContinue) {
+                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "All paths escape so the loop is not necessary.");
+            }
+
+            if (continuous && !blocksmd.anyBreak) {
+                forsmd.methodEscape = true;
+                forsmd.allLast = true;
+            }
+        }
+
+        forsmd.count = 1;
+
+        utility.decrementScope();
+    }
+
+    void processDecl(final DeclContext ctx) {
+        final StatementMetadata declsmd = metadata.getStatementMetadata(ctx);
+
+        final DeclarationContext declctx = ctx.declaration();
+        metadata.createStatementMetadata(declctx);
+        analyzer.visit(declctx);
+
+        declsmd.count = 1;
+    }
+
+    void processContinue(final ContinueContext ctx) {
+        final StatementMetadata continuesmd = metadata.getStatementMetadata(ctx);
+
+        if (!continuesmd.inLoop) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Cannot have a continue statement outside of a loop.");
+        }
+
+        if (continuesmd.lastLoop) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unnecessary continue statement at the end of a loop.");
+        }
+
+        continuesmd.allLast = true;
+        continuesmd.anyContinue = true;
+
+        continuesmd.count = 1;
+    }
+
+    void processBreak(final BreakContext ctx) {
+        final StatementMetadata breaksmd = metadata.getStatementMetadata(ctx);
+
+        if (!breaksmd.inLoop) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Cannot have a break statement outside of a loop.");
+        }
+
+        breaksmd.loopEscape = true;
+        breaksmd.allLast = true;
+        breaksmd.anyBreak = true;
+
+        breaksmd.count = 1;
+    }
+
+    void processReturn(final ReturnContext ctx) {
+        final StatementMetadata returnsmd = metadata.getStatementMetadata(ctx);
+
+        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
+        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
+        expremd.to = definition.objectType;
+        analyzer.visit(exprctx);
+        caster.markCast(expremd);
+
+        returnsmd.methodEscape = true;
+        returnsmd.loopEscape = true;
+        returnsmd.allLast = true;
+
+        returnsmd.count = 1;
+    }
+
+    void processTry(final TryContext ctx) {
+        final StatementMetadata trysmd = metadata.getStatementMetadata(ctx);
+
+        final BlockContext blockctx = ctx.block();
+        final StatementMetadata blocksmd = metadata.createStatementMetadata(blockctx);
+        blocksmd.lastSource = trysmd.lastSource;
+        blocksmd.inLoop = trysmd.inLoop;
+        blocksmd.lastLoop = trysmd.lastLoop;
+        utility.incrementScope();
+        analyzer.visit(blockctx);
+        utility.decrementScope();
+
+        trysmd.methodEscape = blocksmd.methodEscape;
+        trysmd.loopEscape = blocksmd.loopEscape;
+        trysmd.allLast = blocksmd.allLast;
+        trysmd.anyContinue = blocksmd.anyContinue;
+        trysmd.anyBreak = blocksmd.anyBreak;
+
+        int trapcount = 0;
+
+        for (final TrapContext trapctx : ctx.trap()) {
+            final StatementMetadata trapsmd = metadata.createStatementMetadata(trapctx);
+            trapsmd.lastSource = trysmd.lastSource;
+            trapsmd.inLoop = trysmd.inLoop;
+            trapsmd.lastLoop = trysmd.lastLoop;
+            utility.incrementScope();
+            analyzer.visit(trapctx);
+            utility.decrementScope();
+
+            trysmd.methodEscape &= trapsmd.methodEscape;
+            trysmd.loopEscape &= trapsmd.loopEscape;
+            trysmd.allLast &= trapsmd.allLast;
+            trysmd.anyContinue |= trapsmd.anyContinue;
+            trysmd.anyBreak |= trapsmd.anyBreak;
+
+            trapcount = Math.max(trapcount, trapsmd.count);
+        }
+
+        trysmd.count = blocksmd.count + trapcount;
+    }
+
+    void processThrow(final ThrowContext ctx) {
+        final StatementMetadata throwsmd = metadata.getStatementMetadata(ctx);
+
+        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
+        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
+        expremd.to = definition.exceptionType;
+        analyzer.visit(exprctx);
+        caster.markCast(expremd);
+
+        throwsmd.methodEscape = true;
+        throwsmd.loopEscape = true;
+        throwsmd.allLast = true;
+
+        throwsmd.count = 1;
+    }
+
+    void processExpr(final ExprContext ctx) {
+        final StatementMetadata exprsmd = metadata.getStatementMetadata(ctx);
+        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
+        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
+        expremd.read = exprsmd.lastSource;
+        analyzer.visit(exprctx);
+
+        if (!expremd.statement && !exprsmd.lastSource) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Not a statement.");
+        }
+
+        final boolean rtn = exprsmd.lastSource && expremd.from.sort != Sort.VOID;
+        exprsmd.methodEscape = rtn;
+        exprsmd.loopEscape = rtn;
+        exprsmd.allLast = rtn;
+        expremd.to = rtn ? definition.objectType : expremd.from;
+        caster.markCast(expremd);
+
+        exprsmd.count = 1;
+    }
+
+    void processMultiple(final MultipleContext ctx) {
+        final StatementMetadata multiplesmd = metadata.getStatementMetadata(ctx);
+        final List<StatementContext> statectxs = ctx.statement();
+        final StatementContext lastctx = statectxs.get(statectxs.size() - 1);
+
+        for (StatementContext statectx : statectxs) {
+            if (multiplesmd.allLast) {
+                throw new IllegalArgumentException(AnalyzerUtility.error(statectx) +
+                    "Statement will never be executed because all prior paths escape.");
+            }
+
+            final StatementMetadata statesmd = metadata.createStatementMetadata(statectx);
+            statesmd.lastSource = multiplesmd.lastSource && statectx == lastctx;
+            statesmd.inLoop = multiplesmd.inLoop;
+            statesmd.lastLoop = (multiplesmd.beginLoop || multiplesmd.lastLoop) && statectx == lastctx;
+            analyzer.visit(statectx);
+
+            multiplesmd.methodEscape = statesmd.methodEscape;
+            multiplesmd.loopEscape = statesmd.loopEscape;
+            multiplesmd.allLast = statesmd.allLast;
+            multiplesmd.anyContinue |= statesmd.anyContinue;
+            multiplesmd.anyBreak |= statesmd.anyBreak;
+
+            multiplesmd.count += statesmd.count;
+        }
+    }
+
+    void processSingle(final SingleContext ctx) {
+        final StatementMetadata singlesmd = metadata.getStatementMetadata(ctx);
+
+        final StatementContext statectx = ctx.statement();
+        final StatementMetadata statesmd = metadata.createStatementMetadata(statectx);
+        statesmd.lastSource = singlesmd.lastSource;
+        statesmd.inLoop = singlesmd.inLoop;
+        statesmd.lastLoop = singlesmd.beginLoop || singlesmd.lastLoop;
+        analyzer.visit(statectx);
+
+        singlesmd.methodEscape = statesmd.methodEscape;
+        singlesmd.loopEscape = statesmd.loopEscape;
+        singlesmd.allLast = statesmd.allLast;
+        singlesmd.anyContinue = statesmd.anyContinue;
+        singlesmd.anyBreak = statesmd.anyBreak;
+
+        singlesmd.count = statesmd.count;
+    }
+
+    void processInitializer(InitializerContext ctx) {
+        final DeclarationContext declctx = ctx.declaration();
+        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
+
+        if (declctx != null) {
+            metadata.createStatementMetadata(declctx);
+            analyzer.visit(declctx);
+        } else if (exprctx != null) {
+            final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
+            expremd.read = false;
+            analyzer.visit(exprctx);
+
+            expremd.to = expremd.from;
+            caster.markCast(expremd);
+
+            if (!expremd.statement) {
+                throw new IllegalArgumentException(AnalyzerUtility.error(exprctx) +
+                    "The initializer of a for loop must be a statement.");
+            }
+        } else {
+            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+        }
+    }
+
+    void processAfterthought(AfterthoughtContext ctx) {
+        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
+
+        if (exprctx != null) {
+            final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
+            expremd.read = false;
+            analyzer.visit(exprctx);
+
+            expremd.to = expremd.from;
+            caster.markCast(expremd);
+
+            if (!expremd.statement) {
+                throw new IllegalArgumentException(AnalyzerUtility.error(exprctx) +
+                    "The afterthought of a for loop must be a statement.");
+            }
+        }
+    }
+
+    void processDeclaration(final DeclarationContext ctx) {
+        final DecltypeContext decltypectx = ctx.decltype();
+        final ExpressionMetadata decltypeemd = metadata.createExpressionMetadata(decltypectx);
+        analyzer.visit(decltypectx);
+
+        for (final DeclvarContext declvarctx : ctx.declvar()) {
+            final ExpressionMetadata declvaremd = metadata.createExpressionMetadata(declvarctx);
+            declvaremd.to = decltypeemd.from;
+            analyzer.visit(declvarctx);
+        }
+    }
+
+    void processDecltype(final DecltypeContext ctx) {
+        final ExpressionMetadata decltypeemd = metadata.getExpressionMetadata(ctx);
+        final String name = ctx.getText();
+        decltypeemd.from = definition.getType(name);
+    }
+
+    void processDeclvar(final DeclvarContext ctx) {
+        final ExpressionMetadata declvaremd = metadata.getExpressionMetadata(ctx);
+
+        final String name = ctx.ID().getText();
+        declvaremd.postConst = utility.addVariable(ctx, name, declvaremd.to).slot;
+
+        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
+
+        if (exprctx != null) {
+            final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
+            expremd.to = declvaremd.to;
+            analyzer.visit(exprctx);
+            caster.markCast(expremd);
+        }
+    }
+
+    void processTrap(final TrapContext ctx) {
+        final StatementMetadata trapsmd = metadata.getStatementMetadata(ctx);
+
+        final String type = ctx.TYPE().getText();
+        trapsmd.exception = definition.getType(type);
+
+        try {
+            trapsmd.exception.clazz.asSubclass(Exception.class);
+        } catch (final ClassCastException exception) {
+            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Invalid exception type [" + trapsmd.exception.name + "].");
+        }
+
+        final String id = ctx.ID().getText();
+        trapsmd.slot = utility.addVariable(ctx, id, trapsmd.exception).slot;
+
+        final BlockContext blockctx = ctx.block();
+
+        if (blockctx != null) {
+            final StatementMetadata blocksmd = metadata.createStatementMetadata(blockctx);
+            blocksmd.lastSource = trapsmd.lastSource;
+            blocksmd.inLoop = trapsmd.inLoop;
+            blocksmd.lastLoop = trapsmd.lastLoop;
+            analyzer.visit(blockctx);
+
+            trapsmd.methodEscape = blocksmd.methodEscape;
+            trapsmd.loopEscape = blocksmd.loopEscape;
+            trapsmd.allLast = blocksmd.allLast;
+            trapsmd.anyContinue = blocksmd.anyContinue;
+            trapsmd.anyBreak = blocksmd.anyBreak;
+        } else if (ctx.emptyscope() == null) {
+            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerUtility.java
new file mode 100644
index 00000000000..11fb669f190
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerUtility.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.tree.ParseTree;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.PainlessParser.ExpressionContext;
+import org.elasticsearch.painless.PainlessParser.PrecedenceContext;
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.Iterator;
+
+class AnalyzerUtility {
+    static class Variable {
+        final String name;
+        final Type type;
+        final int slot;
+
+        private Variable(final String name, final Type type, final int slot) {
+            this.name = name;
+            this.type = type;
+            this.slot = slot;
+        }
+    }
+
+    /**
+     * A utility method to output consistent error messages.
+     * @param ctx The ANTLR node the error occurred in.
+     * @return The error message with tacked on line number and character position.
+     */
+    static String error(final ParserRuleContext ctx) {
+        return "Analyzer Error [" + ctx.getStart().getLine() + ":" + ctx.getStart().getCharPositionInLine() + "]: ";
+    }
+
+    /**
+     * The ANTLR parse tree is modified in one single case; a parent node needs to check a child node to see if it's
+     * a precedence node, and if so, it must be removed from the tree permanently.  Once the ANTLR tree is built,
+     * precedence nodes are no longer necessary to maintain the correct ordering of the tree, so they only
+     * add a level of indirection where complicated decisions about metadata passing would have to be made.  This
+     * method removes the need for those decisions.
+     * @param source The child ANTLR node to check for precedence.
+     * @return The updated child ANTLR node.
+     */
+    static ExpressionContext updateExpressionTree(ExpressionContext source) {
+        // Check to see if the ANTLR node is a precedence node.
+        if (source instanceof PainlessParser.PrecedenceContext) {
+            final ParserRuleContext parent = source.getParent();
+            int index = 0;
+
+            // Mark the index of the source node within the list of child nodes from the parent.
+            for (final ParseTree child : parent.children) {
+                if (child == source) {
+                    break;
+                }
+
+                ++index;
+            }
+
+            // If there are multiple precedence nodes in a row, remove them all.
+            while (source instanceof PrecedenceContext) {
+                source = ((PrecedenceContext)source).expression();
+            }
+
+            // Update the parent node with the child of the precedence node.
+            parent.children.set(index, source);
+        }
+
+        return source;
+    }
+
+    private final Deque<Integer> scopes = new ArrayDeque<>();
+    private final Deque<Variable> variables = new ArrayDeque<>();
+
+    void incrementScope() {
+        scopes.push(0);
+    }
+
+    void decrementScope() {
+        int remove = scopes.pop();
+
+        while (remove > 0) {
+            variables.pop();
+            --remove;
+        }
+    }
+
+    Variable getVariable(final String name) {
+        final Iterator<Variable> itr = variables.iterator();
+
+        while (itr.hasNext()) {
+            final Variable variable = itr.next();
+
+            if (variable.name.equals(name)) {
+                return variable;
+            }
+        }
+
+        return null;
+    }
+
+    Variable addVariable(final ParserRuleContext source, final String name, final Type type) {
+        if (getVariable(name) != null) {
+            if (source == null) {
+                throw new IllegalArgumentException("Argument name [" + name + "] already defined within the scope.");
+            } else {
+                throw new IllegalArgumentException(error(source) + "Variable name [" + name + "] already defined within the scope.");
+            }
+        }
+
+        final Variable previous = variables.peekFirst();
+        int slot = 0;
+
+        if (previous != null) {
+            slot += previous.slot + previous.type.type.getSize();
+        }
+
+        final Variable variable = new Variable(name, type, slot);
+        variables.push(variable);
+
+        final int update = scopes.pop() + 1;
+        scopes.push(update);
+
+        return variable;
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index d1b0e2dc6fe..3d8123a4800 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -160,7 +160,7 @@ final class Compiler {
             //    throw new RuntimeException(e);
             // }
 
-            final Class<? extends Executable> clazz = loader.define(Writer.CLASS_NAME, bytes);
+            final Class<? extends Executable> clazz = loader.define(WriterConstants.CLASS_NAME, bytes);
             final java.lang.reflect.Constructor<? extends Executable> constructor =
                     clazz.getConstructor(Definition.class, String.class, String.class);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
index cace48ff433..e38d6da7d98 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
@@ -20,11 +20,8 @@
 package org.elasticsearch.painless;
 
 import org.antlr.v4.runtime.ParserRuleContext;
-import org.antlr.v4.runtime.tree.ParseTree;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Type;
-import org.elasticsearch.painless.PainlessParser.ExpressionContext;
-import org.elasticsearch.painless.PainlessParser.PrecedenceContext;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -37,7 +34,6 @@ import java.util.Map;
  * the root of the ANTLR parse tree, and the {@link CompilerSettings}.
  */
 class Metadata {
-
     /**
      * StatementMetadata is used to store metadata mostly about
      * control flow for ANTLR nodes related to if/else, do, while, for, etc.
@@ -386,15 +382,6 @@ class Metadata {
         }
     }
 
-    /**
-     * A utility method to output consistent error messages.
-     * @param ctx The ANTLR node the error occurred in.
-     * @return The error message with tacked on line number and character position.
-     */
-    static String error(final ParserRuleContext ctx) {
-        return "Error [" + ctx.getStart().getLine() + ":" + ctx.getStart().getCharPositionInLine() + "]: ";
-    }
-
     /**
      * Acts as both the Painless API and white-list for what types and methods are allowed.
      */
@@ -490,49 +477,13 @@ class Metadata {
         final StatementMetadata sourcesmd = statementMetadata.get(source);
 
         if (sourcesmd == null) {
-            throw new IllegalStateException(error(source) + "Statement metadata does not exist at" +
+            throw new IllegalStateException("Statement metadata does not exist at" +
                 " the parse node with text [" + source.getText() + "].");
         }
 
         return sourcesmd;
     }
 
-    /**
-     * The ANTLR parse tree is modified in one single case; a parent node needs to check a child node to see if it's
-     * a precedence node, and if so, it must be removed from the tree permanently. Once the ANTLR tree is built,
-     * precedence nodes are no longer necessary to maintain the correct ordering of the tree, so they only
-     * add a level of indirection where complicated decisions about metadata passing would have to be made.  This
-     * method removes the need for those decisions.
-     * @param source The child ANTLR node to check for precedence.
-     * @return The updated child ANTLR node.
-     */
-    ExpressionContext updateExpressionTree(ExpressionContext source) {
-        // Check to see if the ANTLR node is a precedence node.
-        if (source instanceof PrecedenceContext) {
-            final ParserRuleContext parent = source.getParent();
-            int index = 0;
-
-            // Mark the index of the source node within the list of child nodes from the parent.
-            for (final ParseTree child : parent.children) {
-                if (child == source) {
-                    break;
-                }
-
-                ++index;
-            }
-
-            // If there are multiple precedence nodes in a row, remove them all.
-            while (source instanceof PrecedenceContext) {
-                source = ((PrecedenceContext)source).expression();
-            }
-
-            // Update the parent node with the child of the precedence node.
-            parent.children.set(index, source);
-        }
-
-        return source;
-    }
-
     /**
      * Creates a new ExpressionMetadata and stores it in the expressionMetadata map.
      * @param source The ANTLR node for this metadata.
@@ -554,7 +505,7 @@ class Metadata {
         final ExpressionMetadata sourceemd = expressionMetadata.get(source);
 
         if (sourceemd == null) {
-            throw new IllegalStateException(error(source) + "Expression metadata does not exist at" +
+            throw new IllegalStateException("Expression metadata does not exist at" +
                 " the parse node with text [" + source.getText() + "].");
         }
 
@@ -582,7 +533,7 @@ class Metadata {
         final ExternalMetadata sourceemd = externalMetadata.get(source);
 
         if (sourceemd == null) {
-            throw new IllegalStateException(error(source) + "External metadata does not exist at" +
+            throw new IllegalStateException("External metadata does not exist at" +
                 " the parse node with text [" + source.getText() + "].");
         }
 
@@ -610,7 +561,7 @@ class Metadata {
         final ExtNodeMetadata sourceemd = extNodeMetadata.get(source);
 
         if (sourceemd == null) {
-            throw new IllegalStateException(error(source) + "External metadata does not exist at" +
+            throw new IllegalStateException("External metadata does not exist at" +
                 " the parse node with text [" + source.getText() + "].");
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index 431e724127f..4ddb260aea0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -19,20 +19,11 @@
 
 package org.elasticsearch.painless;
 
-import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.tree.ParseTree;
-import org.elasticsearch.painless.Definition.Cast;
-import org.elasticsearch.painless.Definition.Constructor;
-import org.elasticsearch.painless.Definition.Field;
-import org.elasticsearch.painless.Definition.Method;
-import org.elasticsearch.painless.Definition.Sort;
-import org.elasticsearch.painless.Definition.Transform;
-import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.PainlessParser.AfterthoughtContext;
 import org.elasticsearch.painless.PainlessParser.ArgumentsContext;
 import org.elasticsearch.painless.PainlessParser.AssignmentContext;
 import org.elasticsearch.painless.PainlessParser.BinaryContext;
-import org.elasticsearch.painless.PainlessParser.BlockContext;
 import org.elasticsearch.painless.PainlessParser.BoolContext;
 import org.elasticsearch.painless.PainlessParser.BreakContext;
 import org.elasticsearch.painless.PainlessParser.CastContext;
@@ -48,7 +39,6 @@ import org.elasticsearch.painless.PainlessParser.DoContext;
 import org.elasticsearch.painless.PainlessParser.EmptyContext;
 import org.elasticsearch.painless.PainlessParser.EmptyscopeContext;
 import org.elasticsearch.painless.PainlessParser.ExprContext;
-import org.elasticsearch.painless.PainlessParser.ExpressionContext;
 import org.elasticsearch.painless.PainlessParser.ExtbraceContext;
 import org.elasticsearch.painless.PainlessParser.ExtcallContext;
 import org.elasticsearch.painless.PainlessParser.ExtcastContext;
@@ -75,261 +65,82 @@ import org.elasticsearch.painless.PainlessParser.PreincContext;
 import org.elasticsearch.painless.PainlessParser.ReturnContext;
 import org.elasticsearch.painless.PainlessParser.SingleContext;
 import org.elasticsearch.painless.PainlessParser.SourceContext;
-import org.elasticsearch.painless.PainlessParser.StatementContext;
 import org.elasticsearch.painless.PainlessParser.ThrowContext;
 import org.elasticsearch.painless.PainlessParser.TrapContext;
 import org.elasticsearch.painless.PainlessParser.TrueContext;
 import org.elasticsearch.painless.PainlessParser.TryContext;
 import org.elasticsearch.painless.PainlessParser.UnaryContext;
 import org.elasticsearch.painless.PainlessParser.WhileContext;
-import org.elasticsearch.script.ScoreAccessor;
 import org.objectweb.asm.ClassWriter;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.commons.GeneratorAdapter;
 
-import java.lang.invoke.MethodType;
-import java.util.ArrayDeque;
-import java.util.Deque;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.elasticsearch.painless.PainlessParser.ADD;
-import static org.elasticsearch.painless.PainlessParser.BWAND;
-import static org.elasticsearch.painless.PainlessParser.BWOR;
-import static org.elasticsearch.painless.PainlessParser.BWXOR;
-import static org.elasticsearch.painless.PainlessParser.DIV;
-import static org.elasticsearch.painless.PainlessParser.LSH;
-import static org.elasticsearch.painless.PainlessParser.MUL;
-import static org.elasticsearch.painless.PainlessParser.REM;
-import static org.elasticsearch.painless.PainlessParser.RSH;
-import static org.elasticsearch.painless.PainlessParser.SUB;
-import static org.elasticsearch.painless.PainlessParser.USH;
+import static org.elasticsearch.painless.WriterConstants.BASE_CLASS_TYPE;
+import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
+import static org.elasticsearch.painless.WriterConstants.CONSTRUCTOR;
+import static org.elasticsearch.painless.WriterConstants.EXECUTE;
+import static org.elasticsearch.painless.WriterConstants.MAP_GET;
+import static org.elasticsearch.painless.WriterConstants.MAP_TYPE;
+import static org.elasticsearch.painless.WriterConstants.SCORE_ACCESSOR_FLOAT;
+import static org.elasticsearch.painless.WriterConstants.SCORE_ACCESSOR_TYPE;
+import static org.elasticsearch.painless.WriterConstants.SIGNATURE;
 
 class Writer extends PainlessParserBaseVisitor<Void> {
-    private static class Branch {
-        final ParserRuleContext source;
-
-        Label begin = null;
-        Label end = null;
-        Label tru = null;
-        Label fals = null;
-
-        private Branch(final ParserRuleContext source) {
-            this.source = source;
-        }
-    }
-
-    final static String BASE_CLASS_NAME = Executable.class.getName();
-    final static String CLASS_NAME = BASE_CLASS_NAME + "$CompiledPainlessExecutable";
-    private final static org.objectweb.asm.Type BASE_CLASS_TYPE = org.objectweb.asm.Type.getType(Executable.class);
-    private final static org.objectweb.asm.Type CLASS_TYPE = org.objectweb.asm.Type.getType("L" + CLASS_NAME.replace(".", "/") + ";");
-
-    private final static org.objectweb.asm.commons.Method CONSTRUCTOR =
-        getAsmMethod(void.class, "<init>", Definition.class, String.class, String.class);
-    private final static org.objectweb.asm.commons.Method EXECUTE = getAsmMethod(Object.class, "execute", Map.class);
-    private final static String SIGNATURE = "(Ljava/util/Map<Ljava/lang/String;Ljava/lang/Object;>;)Ljava/lang/Object;";
-
-    private final static org.objectweb.asm.Type PAINLESS_ERROR_TYPE = org.objectweb.asm.Type.getType(PainlessError.class);
-
-    private final static org.objectweb.asm.Type DEFINITION_TYPE = org.objectweb.asm.Type.getType(Definition.class);
-
-    private final static org.objectweb.asm.Type MAP_TYPE = org.objectweb.asm.Type.getType(Map.class);
-    private final static org.objectweb.asm.commons.Method MAP_GET = getAsmMethod(Object.class, "get", Object.class);
-
-    private final static org.objectweb.asm.Type SCORE_ACCESSOR_TYPE = org.objectweb.asm.Type.getType(ScoreAccessor.class);
-    private final static org.objectweb.asm.commons.Method SCORE_ACCESSOR_FLOAT = getAsmMethod(float.class, "floatValue");
-
-    private final static org.objectweb.asm.commons.Method DEF_METHOD_CALL = getAsmMethod(
-        Object.class, "methodCall", Object.class, String.class, Definition.class, Object[].class, boolean[].class);
-    private final static org.objectweb.asm.commons.Method DEF_ARRAY_STORE = getAsmMethod(
-        void.class, "arrayStore", Object.class, Object.class, Object.class, Definition.class, boolean.class, boolean.class);
-    private final static org.objectweb.asm.commons.Method DEF_ARRAY_LOAD = getAsmMethod(
-        Object.class, "arrayLoad", Object.class, Object.class, Definition.class, boolean.class);
-    private final static org.objectweb.asm.commons.Method DEF_FIELD_STORE = getAsmMethod(
-        void.class, "fieldStore", Object.class, Object.class, String.class, Definition.class, boolean.class);
-    private final static org.objectweb.asm.commons.Method DEF_FIELD_LOAD = getAsmMethod(
-        Object.class, "fieldLoad", Object.class, String.class, Definition.class);
-
-    private final static org.objectweb.asm.commons.Method DEF_NOT_CALL = getAsmMethod(Object.class, "not", Object.class);
-    private final static org.objectweb.asm.commons.Method DEF_NEG_CALL = getAsmMethod(Object.class, "neg", Object.class);
-    private final static org.objectweb.asm.commons.Method DEF_MUL_CALL = getAsmMethod(Object.class, "mul", Object.class, Object.class);
-    private final static org.objectweb.asm.commons.Method DEF_DIV_CALL = getAsmMethod(Object.class, "div", Object.class, Object.class);
-    private final static org.objectweb.asm.commons.Method DEF_REM_CALL = getAsmMethod(Object.class, "rem", Object.class, Object.class);
-    private final static org.objectweb.asm.commons.Method DEF_ADD_CALL = getAsmMethod(Object.class, "add", Object.class, Object.class);
-    private final static org.objectweb.asm.commons.Method DEF_SUB_CALL = getAsmMethod(Object.class, "sub", Object.class, Object.class);
-    private final static org.objectweb.asm.commons.Method DEF_LSH_CALL = getAsmMethod(Object.class, "lsh", Object.class, Object.class);
-    private final static org.objectweb.asm.commons.Method DEF_RSH_CALL = getAsmMethod(Object.class, "rsh", Object.class, Object.class);
-    private final static org.objectweb.asm.commons.Method DEF_USH_CALL = getAsmMethod(Object.class, "ush", Object.class, Object.class);
-    private final static org.objectweb.asm.commons.Method DEF_AND_CALL = getAsmMethod(Object.class, "and", Object.class, Object.class);
-    private final static org.objectweb.asm.commons.Method DEF_XOR_CALL = getAsmMethod(Object.class, "xor", Object.class, Object.class);
-    private final static org.objectweb.asm.commons.Method DEF_OR_CALL  = getAsmMethod(Object.class, "or" , Object.class, Object.class);
-    private final static org.objectweb.asm.commons.Method DEF_EQ_CALL  = getAsmMethod(boolean.class, "eq" , Object.class, Object.class);
-    private final static org.objectweb.asm.commons.Method DEF_LT_CALL  = getAsmMethod(boolean.class, "lt" , Object.class, Object.class);
-    private final static org.objectweb.asm.commons.Method DEF_LTE_CALL = getAsmMethod(boolean.class, "lte", Object.class, Object.class);
-    private final static org.objectweb.asm.commons.Method DEF_GT_CALL  = getAsmMethod(boolean.class, "gt" , Object.class, Object.class);
-    private final static org.objectweb.asm.commons.Method DEF_GTE_CALL = getAsmMethod(boolean.class, "gte", Object.class, Object.class);
-
-    private final static org.objectweb.asm.Type STRINGBUILDER_TYPE = org.objectweb.asm.Type.getType(StringBuilder.class);
-
-    private final static org.objectweb.asm.commons.Method STRINGBUILDER_CONSTRUCTOR = getAsmMethod(void.class, "<init>");
-    private final static org.objectweb.asm.commons.Method STRINGBUILDER_APPEND_BOOLEAN =
-        getAsmMethod(StringBuilder.class, "append", boolean.class);
-    private final static org.objectweb.asm.commons.Method STRINGBUILDER_APPEND_CHAR =
-        getAsmMethod(StringBuilder.class, "append", char.class);
-    private final static org.objectweb.asm.commons.Method STRINGBUILDER_APPEND_INT =
-        getAsmMethod(StringBuilder.class, "append", int.class);
-    private final static org.objectweb.asm.commons.Method STRINGBUILDER_APPEND_LONG =
-        getAsmMethod(StringBuilder.class, "append", long.class);
-    private final static org.objectweb.asm.commons.Method STRINGBUILDER_APPEND_FLOAT =
-        getAsmMethod(StringBuilder.class, "append", float.class);
-    private final static org.objectweb.asm.commons.Method STRINGBUILDER_APPEND_DOUBLE =
-        getAsmMethod(StringBuilder.class, "append", double.class);
-    private final static org.objectweb.asm.commons.Method STRINGBUILDER_APPEND_STRING =
-        getAsmMethod(StringBuilder.class, "append", String.class);
-    private final static org.objectweb.asm.commons.Method STRINGBUILDER_APPEND_OBJECT =
-        getAsmMethod(StringBuilder.class, "append", Object.class);
-    private final static org.objectweb.asm.commons.Method STRINGBUILDER_TOSTRING = getAsmMethod(String.class, "toString");
-
-    private final static org.objectweb.asm.commons.Method TOINTEXACT_LONG = getAsmMethod(int.class, "toIntExact", long.class);
-    private final static org.objectweb.asm.commons.Method NEGATEEXACT_INT = getAsmMethod(int.class, "negateExact", int.class);
-    private final static org.objectweb.asm.commons.Method NEGATEEXACT_LONG = getAsmMethod(long.class, "negateExact", long.class);
-    private final static org.objectweb.asm.commons.Method MULEXACT_INT = getAsmMethod(int.class, "multiplyExact", int.class, int.class);
-    private final static org.objectweb.asm.commons.Method MULEXACT_LONG = getAsmMethod(long.class, "multiplyExact", long.class, long.class);
-    private final static org.objectweb.asm.commons.Method ADDEXACT_INT = getAsmMethod(int.class, "addExact", int.class, int.class);
-    private final static org.objectweb.asm.commons.Method ADDEXACT_LONG = getAsmMethod(long.class, "addExact", long.class, long.class);
-    private final static org.objectweb.asm.commons.Method SUBEXACT_INT = getAsmMethod(int.class, "subtractExact", int.class, int.class);
-    private final static org.objectweb.asm.commons.Method SUBEXACT_LONG = getAsmMethod(long.class, "subtractExact", long.class, long.class);
-
-    private final static org.objectweb.asm.commons.Method CHECKEQUALS =
-        getAsmMethod(boolean.class, "checkEquals", Object.class, Object.class);
-    private final static org.objectweb.asm.commons.Method TOBYTEEXACT_INT = getAsmMethod(byte.class, "toByteExact", int.class);
-    private final static org.objectweb.asm.commons.Method TOBYTEEXACT_LONG = getAsmMethod(byte.class, "toByteExact", long.class);
-    private final static org.objectweb.asm.commons.Method TOBYTEWOOVERFLOW_FLOAT =
-        getAsmMethod(byte.class, "toByteWithoutOverflow", float.class);
-    private final static org.objectweb.asm.commons.Method TOBYTEWOOVERFLOW_DOUBLE =
-        getAsmMethod(byte.class, "toByteWithoutOverflow", double.class);
-    private final static org.objectweb.asm.commons.Method TOSHORTEXACT_INT = getAsmMethod(short.class, "toShortExact", int.class);
-    private final static org.objectweb.asm.commons.Method TOSHORTEXACT_LONG = getAsmMethod(short.class, "toShortExact", long.class);
-    private final static org.objectweb.asm.commons.Method TOSHORTWOOVERFLOW_FLOAT =
-        getAsmMethod(short.class, "toShortWithoutOverflow", float.class);
-    private final static org.objectweb.asm.commons.Method TOSHORTWOOVERFLOW_DOUBLE =
-        getAsmMethod(short.class, "toShortWihtoutOverflow", double.class);
-    private final static org.objectweb.asm.commons.Method TOCHAREXACT_INT = getAsmMethod(char.class, "toCharExact", int.class);
-    private final static org.objectweb.asm.commons.Method TOCHAREXACT_LONG = getAsmMethod(char.class, "toCharExact", long.class);
-    private final static org.objectweb.asm.commons.Method TOCHARWOOVERFLOW_FLOAT =
-        getAsmMethod(char.class, "toCharWithoutOverflow", float.class);
-    private final static org.objectweb.asm.commons.Method TOCHARWOOVERFLOW_DOUBLE =
-        getAsmMethod(char.class, "toCharWithoutOverflow", double.class);
-    private final static org.objectweb.asm.commons.Method TOINTWOOVERFLOW_FLOAT =
-        getAsmMethod(int.class, "toIntWithoutOverflow", float.class);
-    private final static org.objectweb.asm.commons.Method TOINTWOOVERFLOW_DOUBLE =
-        getAsmMethod(int.class, "toIntWithoutOverflow", double.class);
-    private final static org.objectweb.asm.commons.Method TOLONGWOOVERFLOW_FLOAT =
-        getAsmMethod(long.class, "toLongWithoutOverflow", float.class);
-    private final static org.objectweb.asm.commons.Method TOLONGWOOVERFLOW_DOUBLE =
-        getAsmMethod(long.class, "toLongWithoutOverflow", double.class);
-    private final static org.objectweb.asm.commons.Method TOFLOATWOOVERFLOW_DOUBLE =
-        getAsmMethod(float.class , "toFloatWihtoutOverflow", double.class);
-    private final static org.objectweb.asm.commons.Method MULWOOVERLOW_FLOAT =
-        getAsmMethod(float.class, "multiplyWithoutOverflow", float.class, float.class);
-    private final static org.objectweb.asm.commons.Method MULWOOVERLOW_DOUBLE =
-        getAsmMethod(double.class, "multiplyWithoutOverflow", double.class, double.class);
-    private final static org.objectweb.asm.commons.Method DIVWOOVERLOW_INT =
-        getAsmMethod(int.class, "divideWithoutOverflow", int.class, int.class);
-    private final static org.objectweb.asm.commons.Method DIVWOOVERLOW_LONG =
-        getAsmMethod(long.class, "divideWithoutOverflow", long.class, long.class);
-    private final static org.objectweb.asm.commons.Method DIVWOOVERLOW_FLOAT =
-        getAsmMethod(float.class, "divideWithoutOverflow", float.class, float.class);
-    private final static org.objectweb.asm.commons.Method DIVWOOVERLOW_DOUBLE =
-        getAsmMethod(double.class, "divideWithoutOverflow", double.class, double.class);
-    private final static org.objectweb.asm.commons.Method REMWOOVERLOW_FLOAT =
-        getAsmMethod(float.class, "remainderWithoutOverflow", float.class, float.class);
-    private final static org.objectweb.asm.commons.Method REMWOOVERLOW_DOUBLE =
-        getAsmMethod(double.class, "remainderWithoutOverflow", double.class, double.class);
-    private final static org.objectweb.asm.commons.Method ADDWOOVERLOW_FLOAT =
-        getAsmMethod(float.class, "addWithoutOverflow", float.class, float.class);
-    private final static org.objectweb.asm.commons.Method ADDWOOVERLOW_DOUBLE =
-        getAsmMethod(double.class, "addWithoutOverflow", double.class, double.class);
-    private final static org.objectweb.asm.commons.Method SUBWOOVERLOW_FLOAT =
-        getAsmMethod(float.class, "subtractWithoutOverflow", float.class, float.class);
-    private final static org.objectweb.asm.commons.Method SUBWOOVERLOW_DOUBLE =
-        getAsmMethod(double.class, "subtractWithoutOverflow", double.class, double.class);
-
-    private static org.objectweb.asm.commons.Method getAsmMethod(final Class<?> rtype, final String name, final Class<?>... ptypes) {
-        return new org.objectweb.asm.commons.Method(name, MethodType.methodType(rtype, ptypes).toMethodDescriptorString());
-    }
-
     static byte[] write(Metadata metadata) {
-        Writer writer = new Writer(metadata);
+        final Writer writer = new Writer(metadata);
 
         return writer.getBytes();
     }
 
     private final Metadata metadata;
-    private final Definition definition;
     private final ParseTree root;
     private final String source;
     private final CompilerSettings settings;
 
-    private final Map<ParserRuleContext, Branch> branches = new HashMap<>();
-    private final Deque<Branch> jumps = new ArrayDeque<>();
-    private final Set<ParserRuleContext> strings = new HashSet<>();
+    private final ClassWriter writer;
+    private final GeneratorAdapter execute;
 
-    private ClassWriter writer;
-    private GeneratorAdapter execute;
+    private final WriterStatement statement;
+    private final WriterExpression expression;
+    private final WriterExternal external;
 
     private Writer(final Metadata metadata) {
         this.metadata = metadata;
-        definition = metadata.definition;
         root = metadata.root;
         source = metadata.source;
         settings = metadata.settings;
 
+        writer = new ClassWriter(ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS);
+
         writeBegin();
         writeConstructor();
+
+        execute = new GeneratorAdapter(Opcodes.ACC_PUBLIC, EXECUTE, SIGNATURE, null, writer);
+
+        final WriterUtility utility = new WriterUtility(metadata, execute);
+        final WriterCaster caster = new WriterCaster(execute);
+
+        statement = new WriterStatement(metadata, execute, this, utility);
+        expression = new WriterExpression(metadata, execute, this, utility, caster);
+        external = new WriterExternal(metadata, execute, this, utility, caster);
+
         writeExecute();
         writeEnd();
     }
 
-    private Branch markBranch(final ParserRuleContext source, final ParserRuleContext... nodes) {
-        final Branch branch = new Branch(source);
-
-        for (final ParserRuleContext node : nodes) {
-            branches.put(node, branch);
-        }
-
-        return branch;
-    }
-
-    private void copyBranch(final Branch branch, final ParserRuleContext... nodes) {
-        for (final ParserRuleContext node : nodes) {
-            branches.put(node, branch);
-        }
-    }
-
-    private Branch getBranch(final ParserRuleContext source) {
-        return branches.get(source);
-    }
-
     private void writeBegin() {
-        final int compute = ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS;
         final int version = Opcodes.V1_7;
         final int access = Opcodes.ACC_PUBLIC | Opcodes.ACC_SUPER | Opcodes.ACC_FINAL;
         final String base = BASE_CLASS_TYPE.getInternalName();
         final String name = CLASS_TYPE.getInternalName();
 
-        writer = new ClassWriter(compute);
         writer.visit(version, access, name, null, base, null);
         writer.visitSource(source, null);
     }
 
     private void writeConstructor() {
-        final int access = Opcodes.ACC_PUBLIC;
-        final GeneratorAdapter constructor = new GeneratorAdapter(access, CONSTRUCTOR, null, null, writer);
+        final GeneratorAdapter constructor = new GeneratorAdapter(Opcodes.ACC_PUBLIC, CONSTRUCTOR, null, null, writer);
         constructor.loadThis();
         constructor.loadArgs();
         constructor.invokeConstructor(org.objectweb.asm.Type.getType(Executable.class), CONSTRUCTOR);
@@ -338,9 +149,6 @@ class Writer extends PainlessParserBaseVisitor<Void> {
     }
 
     private void writeExecute() {
-        final int access = Opcodes.ACC_PUBLIC;
-        execute = new GeneratorAdapter(access, EXECUTE, SIGNATURE, null, writer);
-
         final Label fals = new Label();
         final Label end = new Label();
         execute.visitVarInsn(Opcodes.ALOAD, metadata.inputValueSlot);
@@ -364,2011 +172,6 @@ class Writer extends PainlessParserBaseVisitor<Void> {
         execute.endMethod();
     }
 
-    @Override
-    public Void visitSource(final SourceContext ctx) {
-        final Metadata.StatementMetadata sourcesmd = metadata.getStatementMetadata(ctx);
-
-        for (final StatementContext sctx : ctx.statement()) {
-            visit(sctx);
-        }
-
-        if (!sourcesmd.methodEscape) {
-            execute.visitInsn(Opcodes.ACONST_NULL);
-            execute.returnValue();
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitIf(final IfContext ctx) {
-        final ExpressionContext exprctx = ctx.expression();
-        final boolean els = ctx.ELSE() != null;
-        final Branch branch = markBranch(ctx, exprctx);
-        branch.end = new Label();
-        branch.fals = els ? new Label() : branch.end;
-
-        visit(exprctx);
-
-        final BlockContext blockctx0 = ctx.block(0);
-        final Metadata.StatementMetadata blockmd0 = metadata.getStatementMetadata(blockctx0);
-        visit(blockctx0);
-
-        if (els) {
-            if (!blockmd0.allLast) {
-                execute.goTo(branch.end);
-            }
-
-            execute.mark(branch.fals);
-            visit(ctx.block(1));
-        }
-
-        execute.mark(branch.end);
-
-        return null;
-    }
-
-    @Override
-    public Void visitWhile(final WhileContext ctx) {
-        final ExpressionContext exprctx = ctx.expression();
-        final Branch branch = markBranch(ctx, exprctx);
-        branch.begin = new Label();
-        branch.end = new Label();
-        branch.fals = branch.end;
-
-        jumps.push(branch);
-        execute.mark(branch.begin);
-        visit(exprctx);
-
-        final BlockContext blockctx = ctx.block();
-        boolean allLast = false;
-
-        if (blockctx != null) {
-            final Metadata.StatementMetadata blocksmd = metadata.getStatementMetadata(blockctx);
-            allLast = blocksmd.allLast;
-            writeLoopCounter(blocksmd.count > 0 ? blocksmd.count : 1);
-            visit(blockctx);
-        } else if (ctx.empty() != null) {
-            writeLoopCounter(1);
-        } else {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-        }
-
-        if (!allLast) {
-            execute.goTo(branch.begin);
-        }
-
-        execute.mark(branch.end);
-        jumps.pop();
-
-        return null;
-    }
-
-    @Override
-    public Void visitDo(final DoContext ctx) {
-        final ExpressionContext exprctx = ctx.expression();
-        final Branch branch = markBranch(ctx, exprctx);
-        Label start = new Label();
-        branch.begin = new Label();
-        branch.end = new Label();
-        branch.fals = branch.end;
-
-        final BlockContext blockctx = ctx.block();
-        final Metadata.StatementMetadata blocksmd = metadata.getStatementMetadata(blockctx);
-
-        jumps.push(branch);
-        execute.mark(start);
-        visit(blockctx);
-        execute.mark(branch.begin);
-        visit(exprctx);
-        writeLoopCounter(blocksmd.count > 0 ? blocksmd.count : 1);
-        execute.goTo(start);
-        execute.mark(branch.end);
-        jumps.pop();
-
-        return null;
-    }
-
-    @Override
-    public Void visitFor(final ForContext ctx) {
-        final ExpressionContext exprctx = ctx.expression();
-        final AfterthoughtContext atctx = ctx.afterthought();
-        final Branch branch = markBranch(ctx, exprctx);
-        final Label start = new Label();
-        branch.begin = atctx == null ? start : new Label();
-        branch.end = new Label();
-        branch.fals = branch.end;
-
-        jumps.push(branch);
-
-        if (ctx.initializer() != null) {
-            visit(ctx.initializer());
-        }
-
-        execute.mark(start);
-
-        if (exprctx != null) {
-            visit(exprctx);
-        }
-
-        final BlockContext blockctx = ctx.block();
-        boolean allLast = false;
-
-        if (blockctx != null) {
-            Metadata.StatementMetadata blocksmd = metadata.getStatementMetadata(blockctx);
-            allLast = blocksmd.allLast;
-
-            int count = blocksmd.count > 0 ? blocksmd.count : 1;
-
-            if (atctx != null) {
-                ++count;
-            }
-
-            writeLoopCounter(count);
-            visit(blockctx);
-        } else if (ctx.empty() != null) {
-            writeLoopCounter(1);
-        } else {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-        }
-
-        if (atctx != null) {
-            execute.mark(branch.begin);
-            visit(atctx);
-        }
-
-        if (atctx != null || !allLast) {
-            execute.goTo(start);
-        }
-
-        execute.mark(branch.end);
-        jumps.pop();
-
-        return null;
-    }
-
-    @Override
-    public Void visitDecl(final DeclContext ctx) {
-        visit(ctx.declaration());
-
-        return null;
-    }
-
-    @Override
-    public Void visitContinue(final ContinueContext ctx) {
-        final Branch jump = jumps.peek();
-        execute.goTo(jump.begin);
-
-        return null;
-    }
-
-    @Override
-    public Void visitBreak(final BreakContext ctx) {
-        final Branch jump = jumps.peek();
-        execute.goTo(jump.end);
-
-        return null;
-    }
-
-    @Override
-    public Void visitReturn(final ReturnContext ctx) {
-        visit(ctx.expression());
-        execute.returnValue();
-
-        return null;
-    }
-
-    @Override
-    public Void visitTry(final TryContext ctx) {
-        final TrapContext[] trapctxs = new TrapContext[ctx.trap().size()];
-        ctx.trap().toArray(trapctxs);
-        final Branch branch = markBranch(ctx, trapctxs);
-
-        Label end = new Label();
-        branch.begin = new Label();
-        branch.end = new Label();
-        branch.tru = trapctxs.length > 1 ? end : null;
-
-        execute.mark(branch.begin);
-
-        final BlockContext blockctx = ctx.block();
-        final Metadata.StatementMetadata blocksmd = metadata.getStatementMetadata(blockctx);
-        visit(blockctx);
-
-        if (!blocksmd.allLast) {
-            execute.goTo(end);
-        }
-
-        execute.mark(branch.end);
-
-        for (final TrapContext trapctx : trapctxs) {
-            visit(trapctx);
-        }
-
-        if (!blocksmd.allLast || trapctxs.length > 1) {
-            execute.mark(end);
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitThrow(final ThrowContext ctx) {
-        visit(ctx.expression());
-        execute.throwException();
-
-        return null;
-    }
-
-    @Override
-    public Void visitExpr(final ExprContext ctx) {
-        final Metadata.StatementMetadata exprsmd = metadata.getStatementMetadata(ctx);
-        final ExpressionContext exprctx = ctx.expression();
-        final Metadata.ExpressionMetadata expremd = metadata.getExpressionMetadata(exprctx);
-        visit(exprctx);
-
-        if (exprsmd.methodEscape) {
-            execute.returnValue();
-        } else {
-            writePop(expremd.to.type.getSize());
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitMultiple(final MultipleContext ctx) {
-        for (final StatementContext sctx : ctx.statement()) {
-            visit(sctx);
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitSingle(final SingleContext ctx) {
-        visit(ctx.statement());
-
-        return null;
-    }
-
-    @Override
-    public Void visitEmpty(final EmptyContext ctx) {
-        throw new UnsupportedOperationException(Metadata.error(ctx) + "Unexpected writer state.");
-    }
-
-    @Override
-    public Void visitInitializer(InitializerContext ctx) {
-        final DeclarationContext declctx = ctx.declaration();
-        final ExpressionContext exprctx = ctx.expression();
-
-        if (declctx != null) {
-            visit(declctx);
-        } else if (exprctx != null) {
-            final Metadata.ExpressionMetadata expremd = metadata.getExpressionMetadata(exprctx);
-            visit(exprctx);
-            writePop(expremd.to.type.getSize());
-        } else {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitAfterthought(AfterthoughtContext ctx) {
-        final ExpressionContext exprctx = ctx.expression();
-        final Metadata.ExpressionMetadata expremd = metadata.getExpressionMetadata(exprctx);
-        visit(ctx.expression());
-        writePop(expremd.to.type.getSize());
-
-        return null;
-    }
-
-    @Override
-    public Void visitDeclaration(DeclarationContext ctx) {
-        for (final DeclvarContext declctx : ctx.declvar()) {
-            visit(declctx);
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitDecltype(final DecltypeContext ctx) {
-        throw new UnsupportedOperationException(Metadata.error(ctx) + "Unexpected writer state.");
-    }
-
-    @Override
-    public Void visitDeclvar(final DeclvarContext ctx) {
-        final Metadata.ExpressionMetadata declvaremd = metadata.getExpressionMetadata(ctx);
-        final org.objectweb.asm.Type type = declvaremd.to.type;
-        final Sort sort = declvaremd.to.sort;
-        final int slot = (int)declvaremd.postConst;
-
-        final ExpressionContext exprctx = ctx.expression();
-        final boolean initialize = exprctx == null;
-
-        if (!initialize) {
-            visit(exprctx);
-        }
-
-        switch (sort) {
-            case VOID:   throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-            case BOOL:
-            case BYTE:
-            case SHORT:
-            case CHAR:
-            case INT:    if (initialize) execute.push(0);    break;
-            case LONG:   if (initialize) execute.push(0L);   break;
-            case FLOAT:  if (initialize) execute.push(0.0F); break;
-            case DOUBLE: if (initialize) execute.push(0.0);  break;
-            default:     if (initialize) execute.visitInsn(Opcodes.ACONST_NULL);
-        }
-
-        execute.visitVarInsn(type.getOpcode(Opcodes.ISTORE), slot);
-
-        return null;
-    }
-
-    @Override
-    public Void visitTrap(final TrapContext ctx) {
-        final Metadata.StatementMetadata trapsmd = metadata.getStatementMetadata(ctx);
-
-        final Branch branch = getBranch(ctx);
-        final Label jump = new Label();
-
-        final BlockContext blockctx = ctx.block();
-        final EmptyscopeContext emptyctx = ctx.emptyscope();
-
-        execute.mark(jump);
-        writeLoadStoreVariable(ctx, true, trapsmd.exception, trapsmd.slot);
-
-        if (blockctx != null) {
-            visit(ctx.block());
-        } else if (emptyctx == null) {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-        }
-
-        execute.visitTryCatchBlock(branch.begin, branch.end, jump, trapsmd.exception.type.getInternalName());
-
-        if (branch.tru != null && !trapsmd.allLast) {
-            execute.goTo(branch.tru);
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitPrecedence(final PrecedenceContext ctx) {
-        throw new UnsupportedOperationException(Metadata.error(ctx) + "Unexpected writer state.");
-    }
-
-    @Override
-    public Void visitNumeric(final NumericContext ctx) {
-        final Metadata.ExpressionMetadata numericemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = numericemd.postConst;
-
-        if (postConst == null) {
-            writeNumeric(ctx, numericemd.preConst);
-            checkWriteCast(numericemd);
-        } else {
-            writeConstant(ctx, postConst);
-        }
-
-        checkWriteBranch(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitChar(final CharContext ctx) {
-        final Metadata.ExpressionMetadata charemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = charemd.postConst;
-
-        if (postConst == null) {
-            writeNumeric(ctx, (int)(char)charemd.preConst);
-            checkWriteCast(charemd);
-        } else {
-            writeConstant(ctx, postConst);
-        }
-
-        checkWriteBranch(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitTrue(final TrueContext ctx) {
-        final Metadata.ExpressionMetadata trueemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = trueemd.postConst;
-        final Branch branch = getBranch(ctx);
-
-        if (branch == null) {
-            if (postConst == null) {
-                writeBoolean(ctx, true);
-                checkWriteCast(trueemd);
-            } else {
-                writeConstant(ctx, postConst);
-            }
-        } else if (branch.tru != null) {
-            execute.goTo(branch.tru);
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitFalse(final FalseContext ctx) {
-        final Metadata.ExpressionMetadata falseemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = falseemd.postConst;
-        final Branch branch = getBranch(ctx);
-
-        if (branch == null) {
-            if (postConst == null) {
-                writeBoolean(ctx, false);
-                checkWriteCast(falseemd);
-            } else {
-                writeConstant(ctx, postConst);
-            }
-        } else if (branch.fals != null) {
-            execute.goTo(branch.fals);
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitNull(final NullContext ctx) {
-        final Metadata.ExpressionMetadata nullemd = metadata.getExpressionMetadata(ctx);
-
-        execute.visitInsn(Opcodes.ACONST_NULL);
-        checkWriteCast(nullemd);
-        checkWriteBranch(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExternal(final ExternalContext ctx) {
-        final Metadata.ExpressionMetadata expremd = metadata.getExpressionMetadata(ctx);
-        visit(ctx.extstart());
-        checkWriteCast(expremd);
-        checkWriteBranch(ctx);
-
-        return null;
-    }
-
-
-    @Override
-    public Void visitPostinc(final PostincContext ctx) {
-        final Metadata.ExpressionMetadata expremd = metadata.getExpressionMetadata(ctx);
-        visit(ctx.extstart());
-        checkWriteCast(expremd);
-        checkWriteBranch(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitPreinc(final PreincContext ctx) {
-        final Metadata.ExpressionMetadata expremd = metadata.getExpressionMetadata(ctx);
-        visit(ctx.extstart());
-        checkWriteCast(expremd);
-        checkWriteBranch(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitUnary(final UnaryContext ctx) {
-        final Metadata.ExpressionMetadata unaryemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = unaryemd.postConst;
-        final Object preConst = unaryemd.preConst;
-        final Branch branch = getBranch(ctx);
-
-        if (postConst != null) {
-            if (ctx.BOOLNOT() != null) {
-                if (branch == null) {
-                    writeConstant(ctx, postConst);
-                } else {
-                    if ((boolean)postConst && branch.tru != null) {
-                        execute.goTo(branch.tru);
-                    } else if (!(boolean)postConst && branch.fals != null) {
-                        execute.goTo(branch.fals);
-                    }
-                }
-            } else {
-                writeConstant(ctx, postConst);
-                checkWriteBranch(ctx);
-            }
-        } else if (preConst != null) {
-            if (branch == null) {
-                writeConstant(ctx, preConst);
-                checkWriteCast(unaryemd);
-            } else {
-                throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-            }
-        } else {
-            final ExpressionContext exprctx = ctx.expression();
-
-            if (ctx.BOOLNOT() != null) {
-                final Branch local = markBranch(ctx, exprctx);
-
-                if (branch == null) {
-                    local.fals = new Label();
-                    final Label aend = new Label();
-
-                    visit(exprctx);
-
-                    execute.push(false);
-                    execute.goTo(aend);
-                    execute.mark(local.fals);
-                    execute.push(true);
-                    execute.mark(aend);
-
-                    checkWriteCast(unaryemd);
-                } else {
-                    local.tru = branch.fals;
-                    local.fals = branch.tru;
-
-                    visit(exprctx);
-                }
-            } else {
-                final org.objectweb.asm.Type type = unaryemd.from.type;
-                final Sort sort = unaryemd.from.sort;
-
-                visit(exprctx);
-
-                if (ctx.BWNOT() != null) {
-                    if (sort == Sort.DEF) {
-                        execute.invokeStatic(definition.defobjType.type, DEF_NOT_CALL);
-                    } else {
-                        if (sort == Sort.INT) {
-                            writeConstant(ctx, -1);
-                        } else if (sort == Sort.LONG) {
-                            writeConstant(ctx, -1L);
-                        } else {
-                            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-                        }
-
-                        execute.math(GeneratorAdapter.XOR, type);
-                    }
-                } else if (ctx.SUB() != null) {
-                    if (sort == Sort.DEF) {
-                        execute.invokeStatic(definition.defobjType.type, DEF_NEG_CALL);
-                    } else {
-                        if (settings.getNumericOverflow()) {
-                            execute.math(GeneratorAdapter.NEG, type);
-                        } else {
-                            if (sort == Sort.INT) {
-                                execute.invokeStatic(definition.mathType.type, NEGATEEXACT_INT);
-                            } else if (sort == Sort.LONG) {
-                                execute.invokeStatic(definition.mathType.type, NEGATEEXACT_LONG);
-                            } else {
-                                throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-                            }
-                        }
-                    }
-                } else if (ctx.ADD() == null) {
-                    throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-                }
-
-                checkWriteCast(unaryemd);
-                checkWriteBranch(ctx);
-            }
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitCast(final CastContext ctx) {
-        final Metadata.ExpressionMetadata castemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = castemd.postConst;
-
-        if (postConst == null) {
-            visit(ctx.expression());
-            checkWriteCast(castemd);
-        } else {
-            writeConstant(ctx, postConst);
-        }
-
-        checkWriteBranch(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitBinary(final BinaryContext ctx) {
-        final Metadata.ExpressionMetadata binaryemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = binaryemd.postConst;
-        final Object preConst = binaryemd.preConst;
-        final Branch branch = getBranch(ctx);
-
-        if (postConst != null) {
-            writeConstant(ctx, postConst);
-        } else if (preConst != null) {
-            if (branch == null) {
-                writeConstant(ctx, preConst);
-                checkWriteCast(binaryemd);
-            } else {
-                throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-            }
-        } else if (binaryemd.from.sort == Sort.STRING) {
-            final boolean marked = strings.contains(ctx);
-
-            if (!marked) {
-                writeNewStrings();
-            }
-
-            final ExpressionContext exprctx0 = ctx.expression(0);
-            final Metadata.ExpressionMetadata expremd0 = metadata.getExpressionMetadata(exprctx0);
-            strings.add(exprctx0);
-            visit(exprctx0);
-
-            if (strings.contains(exprctx0)) {
-                writeAppendStrings(expremd0.from.sort);
-                strings.remove(exprctx0);
-            }
-
-            final ExpressionContext exprctx1 = ctx.expression(1);
-            final Metadata.ExpressionMetadata expremd1 = metadata.getExpressionMetadata(exprctx1);
-            strings.add(exprctx1);
-            visit(exprctx1);
-
-            if (strings.contains(exprctx1)) {
-                writeAppendStrings(expremd1.from.sort);
-                strings.remove(exprctx1);
-            }
-
-            if (marked) {
-                strings.remove(ctx);
-            } else {
-                writeToStrings();
-            }
-
-            checkWriteCast(binaryemd);
-        } else {
-            final ExpressionContext exprctx0 = ctx.expression(0);
-            final ExpressionContext exprctx1 = ctx.expression(1);
-
-            visit(exprctx0);
-            visit(exprctx1);
-
-            final Type type = binaryemd.from;
-
-            if      (ctx.MUL()   != null) writeBinaryInstruction(ctx, type, MUL);
-            else if (ctx.DIV()   != null) writeBinaryInstruction(ctx, type, DIV);
-            else if (ctx.REM()   != null) writeBinaryInstruction(ctx, type, REM);
-            else if (ctx.ADD()   != null) writeBinaryInstruction(ctx, type, ADD);
-            else if (ctx.SUB()   != null) writeBinaryInstruction(ctx, type, SUB);
-            else if (ctx.LSH()   != null) writeBinaryInstruction(ctx, type, LSH);
-            else if (ctx.USH()   != null) writeBinaryInstruction(ctx, type, USH);
-            else if (ctx.RSH()   != null) writeBinaryInstruction(ctx, type, RSH);
-            else if (ctx.BWAND() != null) writeBinaryInstruction(ctx, type, BWAND);
-            else if (ctx.BWXOR() != null) writeBinaryInstruction(ctx, type, BWXOR);
-            else if (ctx.BWOR()  != null) writeBinaryInstruction(ctx, type, BWOR);
-            else {
-                throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-            }
-
-            checkWriteCast(binaryemd);
-        }
-
-        checkWriteBranch(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitComp(final CompContext ctx) {
-        final Metadata.ExpressionMetadata compemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = compemd.postConst;
-        final Object preConst = compemd.preConst;
-        final Branch branch = getBranch(ctx);
-
-        if (postConst != null) {
-            if (branch == null) {
-                writeConstant(ctx, postConst);
-            } else {
-                if ((boolean)postConst && branch.tru != null) {
-                    execute.mark(branch.tru);
-                } else if (!(boolean)postConst && branch.fals != null) {
-                    execute.mark(branch.fals);
-                }
-            }
-        } else if (preConst != null) {
-            if (branch == null) {
-                writeConstant(ctx, preConst);
-                checkWriteCast(compemd);
-            } else {
-                throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-            }
-        } else {
-            final ExpressionContext exprctx0 = ctx.expression(0);
-            final Metadata.ExpressionMetadata expremd0 = metadata.getExpressionMetadata(exprctx0);
-
-            final ExpressionContext exprctx1 = ctx.expression(1);
-            final Metadata.ExpressionMetadata expremd1 = metadata.getExpressionMetadata(exprctx1);
-            final org.objectweb.asm.Type type = expremd1.to.type;
-            final Sort sort1 = expremd1.to.sort;
-
-            visit(exprctx0);
-
-            if (!expremd1.isNull) {
-                visit(exprctx1);
-            }
-
-            final boolean tru = branch != null && branch.tru != null;
-            final boolean fals = branch != null && branch.fals != null;
-            final Label jump = tru ? branch.tru : fals ? branch.fals : new Label();
-            final Label end = new Label();
-
-            final boolean eq = (ctx.EQ() != null || ctx.EQR() != null) && (tru || !fals) ||
-                (ctx.NE() != null || ctx.NER() != null) && fals;
-            final boolean ne = (ctx.NE() != null || ctx.NER() != null) && (tru || !fals) ||
-                (ctx.EQ() != null || ctx.EQR() != null) && fals;
-            final boolean lt  = ctx.LT()  != null && (tru || !fals) || ctx.GTE() != null && fals;
-            final boolean lte = ctx.LTE() != null && (tru || !fals) || ctx.GT()  != null && fals;
-            final boolean gt  = ctx.GT()  != null && (tru || !fals) || ctx.LTE() != null && fals;
-            final boolean gte = ctx.GTE() != null && (tru || !fals) || ctx.LT()  != null && fals;
-
-            boolean writejump = true;
-
-            switch (sort1) {
-                case VOID:
-                case BYTE:
-                case SHORT:
-                case CHAR:
-                    throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-                case BOOL:
-                    if      (eq) execute.ifZCmp(GeneratorAdapter.EQ, jump);
-                    else if (ne) execute.ifZCmp(GeneratorAdapter.NE, jump);
-                    else {
-                        throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-                    }
-
-                    break;
-                case INT:
-                case LONG:
-                case FLOAT:
-                case DOUBLE:
-                    if      (eq)  execute.ifCmp(type, GeneratorAdapter.EQ, jump);
-                    else if (ne)  execute.ifCmp(type, GeneratorAdapter.NE, jump);
-                    else if (lt)  execute.ifCmp(type, GeneratorAdapter.LT, jump);
-                    else if (lte) execute.ifCmp(type, GeneratorAdapter.LE, jump);
-                    else if (gt)  execute.ifCmp(type, GeneratorAdapter.GT, jump);
-                    else if (gte) execute.ifCmp(type, GeneratorAdapter.GE, jump);
-                    else {
-                        throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-                    }
-
-                    break;
-                case DEF:
-                    if (eq) {
-                        if (expremd1.isNull) {
-                            execute.ifNull(jump);
-                        } else if (!expremd0.isNull && ctx.EQ() != null) {
-                            execute.invokeStatic(definition.defobjType.type, DEF_EQ_CALL);
-                        } else {
-                            execute.ifCmp(type, GeneratorAdapter.EQ, jump);
-                        }
-                    } else if (ne) {
-                        if (expremd1.isNull) {
-                            execute.ifNonNull(jump);
-                        } else if (!expremd0.isNull && ctx.NE() != null) {
-                            execute.invokeStatic(definition.defobjType.type, DEF_EQ_CALL);
-                            execute.ifZCmp(GeneratorAdapter.EQ, jump);
-                        } else {
-                            execute.ifCmp(type, GeneratorAdapter.NE, jump);
-                        }
-                    } else if (lt) {
-                        execute.invokeStatic(definition.defobjType.type, DEF_LT_CALL);
-                    } else if (lte) {
-                        execute.invokeStatic(definition.defobjType.type, DEF_LTE_CALL);
-                    } else if (gt) {
-                        execute.invokeStatic(definition.defobjType.type, DEF_GT_CALL);
-                    } else if (gte) {
-                        execute.invokeStatic(definition.defobjType.type, DEF_GTE_CALL);
-                    } else {
-                        throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-                    }
-
-                    writejump = expremd1.isNull || ne || ctx.EQR() != null;
-
-                    if (branch != null && !writejump) {
-                        execute.ifZCmp(GeneratorAdapter.NE, jump);
-                    }
-
-                    break;
-                default:
-                    if (eq) {
-                        if (expremd1.isNull) {
-                            execute.ifNull(jump);
-                        } else if (ctx.EQ() != null) {
-                            execute.invokeStatic(definition.utilityType.type, CHECKEQUALS);
-
-                            if (branch != null) {
-                                execute.ifZCmp(GeneratorAdapter.NE, jump);
-                            }
-
-                            writejump = false;
-                        } else {
-                            execute.ifCmp(type, GeneratorAdapter.EQ, jump);
-                        }
-                    } else if (ne) {
-                        if (expremd1.isNull) {
-                            execute.ifNonNull(jump);
-                        } else if (ctx.NE() != null) {
-                            execute.invokeStatic(definition.utilityType.type, CHECKEQUALS);
-                            execute.ifZCmp(GeneratorAdapter.EQ, jump);
-                        } else {
-                            execute.ifCmp(type, GeneratorAdapter.NE, jump);
-                        }
-                    } else {
-                        throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-                    }
-            }
-
-            if (branch == null) {
-                if (writejump) {
-                    execute.push(false);
-                    execute.goTo(end);
-                    execute.mark(jump);
-                    execute.push(true);
-                    execute.mark(end);
-                }
-
-                checkWriteCast(compemd);
-            }
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitBool(final BoolContext ctx) {
-        final Metadata.ExpressionMetadata boolemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = boolemd.postConst;
-        final Object preConst = boolemd.preConst;
-        final Branch branch = getBranch(ctx);
-
-        if (postConst != null) {
-            if (branch == null) {
-                writeConstant(ctx, postConst);
-            } else {
-                if ((boolean)postConst && branch.tru != null) {
-                    execute.mark(branch.tru);
-                } else if (!(boolean)postConst && branch.fals != null) {
-                    execute.mark(branch.fals);
-                }
-            }
-        } else if (preConst != null) {
-            if (branch == null) {
-                writeConstant(ctx, preConst);
-                checkWriteCast(boolemd);
-            } else {
-                throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-            }
-        } else {
-            final ExpressionContext exprctx0 = ctx.expression(0);
-            final ExpressionContext exprctx1 = ctx.expression(1);
-
-            if (branch == null) {
-                if (ctx.BOOLAND() != null) {
-                    final Branch local = markBranch(ctx, exprctx0, exprctx1);
-                    local.fals = new Label();
-                    final Label end = new Label();
-
-                    visit(exprctx0);
-                    visit(exprctx1);
-
-                    execute.push(true);
-                    execute.goTo(end);
-                    execute.mark(local.fals);
-                    execute.push(false);
-                    execute.mark(end);
-                } else if (ctx.BOOLOR() != null) {
-                    final Branch branch0 = markBranch(ctx, exprctx0);
-                    branch0.tru = new Label();
-                    final Branch branch1 = markBranch(ctx, exprctx1);
-                    branch1.fals = new Label();
-                    final Label aend = new Label();
-
-                    visit(exprctx0);
-                    visit(exprctx1);
-
-                    execute.mark(branch0.tru);
-                    execute.push(true);
-                    execute.goTo(aend);
-                    execute.mark(branch1.fals);
-                    execute.push(false);
-                    execute.mark(aend);
-                } else {
-                    throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-                }
-
-                checkWriteCast(boolemd);
-            } else {
-                if (ctx.BOOLAND() != null) {
-                    final Branch branch0 = markBranch(ctx, exprctx0);
-                    branch0.fals = branch.fals == null ? new Label() : branch.fals;
-                    final Branch branch1 = markBranch(ctx, exprctx1);
-                    branch1.tru = branch.tru;
-                    branch1.fals = branch.fals;
-
-                    visit(exprctx0);
-                    visit(exprctx1);
-
-                    if (branch.fals == null) {
-                        execute.mark(branch0.fals);
-                    }
-                } else if (ctx.BOOLOR() != null) {
-                    final Branch branch0 = markBranch(ctx, exprctx0);
-                    branch0.tru = branch.tru == null ? new Label() : branch.tru;
-                    final Branch branch1 = markBranch(ctx, exprctx1);
-                    branch1.tru = branch.tru;
-                    branch1.fals = branch.fals;
-
-                    visit(exprctx0);
-                    visit(exprctx1);
-
-                    if (branch.tru == null) {
-                        execute.mark(branch0.tru);
-                    }
-                } else {
-                    throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-                }
-            }
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitConditional(final ConditionalContext ctx) {
-        final Metadata.ExpressionMetadata condemd = metadata.getExpressionMetadata(ctx);
-        final Branch branch = getBranch(ctx);
-
-        final ExpressionContext expr0 = ctx.expression(0);
-        final ExpressionContext expr1 = ctx.expression(1);
-        final ExpressionContext expr2 = ctx.expression(2);
-
-        final Branch local = markBranch(ctx, expr0);
-        local.fals = new Label();
-        local.end = new Label();
-
-        if (branch != null) {
-            copyBranch(branch, expr1, expr2);
-        }
-
-        visit(expr0);
-        visit(expr1);
-        execute.goTo(local.end);
-        execute.mark(local.fals);
-        visit(expr2);
-        execute.mark(local.end);
-
-        if (branch == null) {
-            checkWriteCast(condemd);
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitAssignment(final AssignmentContext ctx) {
-        final Metadata.ExpressionMetadata expremd = metadata.getExpressionMetadata(ctx);
-        visit(ctx.extstart());
-        checkWriteCast(expremd);
-        checkWriteBranch(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtstart(ExtstartContext ctx) {
-        final Metadata.ExternalMetadata startemd = metadata.getExternalMetadata(ctx);
-
-        if (startemd.token == ADD) {
-            final Metadata.ExpressionMetadata storeemd = metadata.getExpressionMetadata(startemd.storeExpr);
-
-            if (startemd.current.sort == Sort.STRING || storeemd.from.sort == Sort.STRING) {
-                writeNewStrings();
-                strings.add(startemd.storeExpr);
-            }
-        }
-
-        final ExtprecContext precctx = ctx.extprec();
-        final ExtcastContext castctx = ctx.extcast();
-        final ExttypeContext typectx = ctx.exttype();
-        final ExtvarContext varctx = ctx.extvar();
-        final ExtnewContext newctx = ctx.extnew();
-        final ExtstringContext stringctx = ctx.extstring();
-
-        if (precctx != null) {
-            visit(precctx);
-        } else if (castctx != null) {
-            visit(castctx);
-        } else if (typectx != null) {
-            visit(typectx);
-        } else if (varctx != null) {
-            visit(varctx);
-        } else if (newctx != null) {
-            visit(newctx);
-        } else if (stringctx != null) {
-            visit(stringctx);
-        } else {
-            throw new IllegalStateException();
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtprec(final ExtprecContext ctx) {
-        final ExtprecContext precctx = ctx.extprec();
-        final ExtcastContext castctx = ctx.extcast();
-        final ExttypeContext typectx = ctx.exttype();
-        final ExtvarContext varctx = ctx.extvar();
-        final ExtnewContext newctx = ctx.extnew();
-        final ExtstringContext stringctx = ctx.extstring();
-
-        if (precctx != null) {
-            visit(precctx);
-        } else if (castctx != null) {
-            visit(castctx);
-        } else if (typectx != null) {
-            visit(typectx);
-        } else if (varctx != null) {
-            visit(varctx);
-        } else if (newctx != null) {
-            visit(newctx);
-        } else if (stringctx != null) {
-            visit(stringctx);
-        } else {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-        }
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        if (dotctx != null) {
-            visit(dotctx);
-        } else if (bracectx != null) {
-            visit(bracectx);
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtcast(final ExtcastContext ctx) {
-        Metadata.ExtNodeMetadata castenmd = metadata.getExtNodeMetadata(ctx);
-
-        final ExtprecContext precctx = ctx.extprec();
-        final ExtcastContext castctx = ctx.extcast();
-        final ExttypeContext typectx = ctx.exttype();
-        final ExtvarContext varctx = ctx.extvar();
-        final ExtnewContext newctx = ctx.extnew();
-        final ExtstringContext stringctx = ctx.extstring();
-
-        if (precctx != null) {
-            visit(precctx);
-        } else if (castctx != null) {
-            visit(castctx);
-        } else if (typectx != null) {
-            visit(typectx);
-        } else if (varctx != null) {
-            visit(varctx);
-        } else if (newctx != null) {
-            visit(newctx);
-        } else if (stringctx != null) {
-            visit(stringctx);
-        } else {
-            throw new IllegalStateException(Metadata.error(ctx) + "Unexpected writer state.");
-        }
-
-        checkWriteCast(ctx, castenmd.castTo);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtbrace(final ExtbraceContext ctx) {
-        final ExpressionContext exprctx = metadata.updateExpressionTree(ctx.expression());
-
-        visit(exprctx);
-        writeLoadStoreExternal(ctx);
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        if (dotctx != null) {
-            visit(dotctx);
-        } else if (bracectx != null) {
-            visit(bracectx);
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtdot(final ExtdotContext ctx) {
-        final ExtcallContext callctx = ctx.extcall();
-        final ExtfieldContext fieldctx = ctx.extfield();
-
-        if (callctx != null) {
-            visit(callctx);
-        } else if (fieldctx != null) {
-            visit(fieldctx);
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitExttype(final ExttypeContext ctx) {
-        visit(ctx.extdot());
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtcall(final ExtcallContext ctx) {
-        writeCallExternal(ctx);
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        if (dotctx != null) {
-            visit(dotctx);
-        } else if (bracectx != null) {
-            visit(bracectx);
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtvar(final ExtvarContext ctx) {
-        writeLoadStoreExternal(ctx);
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        if (dotctx != null) {
-            visit(dotctx);
-        } else if (bracectx != null) {
-            visit(bracectx);
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtfield(final ExtfieldContext ctx) {
-        writeLoadStoreExternal(ctx);
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        if (dotctx != null) {
-            visit(dotctx);
-        } else if (bracectx != null) {
-            visit(bracectx);
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtnew(ExtnewContext ctx) {
-        writeNewExternal(ctx);
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        if (dotctx != null) {
-            visit(dotctx);
-        } else if (bracectx != null) {
-            visit(bracectx);
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtstring(ExtstringContext ctx) {
-        final Metadata.ExtNodeMetadata stringenmd = metadata.getExtNodeMetadata(ctx);
-
-        writeConstant(ctx, stringenmd.target);
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        if (dotctx != null) {
-            visit(dotctx);
-        } else if (bracectx != null) {
-            visit(bracectx);
-        }
-
-        return null;
-    }
-
-    @Override
-    public Void visitArguments(final ArgumentsContext ctx) {
-        throw new UnsupportedOperationException(Metadata.error(ctx) + "Unexpected writer state.");
-    }
-
-    @Override
-    public Void visitIncrement(IncrementContext ctx) {
-        final Metadata.ExpressionMetadata incremd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = incremd.postConst;
-
-        if (postConst == null) {
-            writeNumeric(ctx, incremd.preConst);
-            checkWriteCast(incremd);
-        } else {
-            writeConstant(ctx, postConst);
-        }
-
-        checkWriteBranch(ctx);
-
-        return null;
-    }
-
-    private void writeLoopCounter(final int count) {
-        final Label end = new Label();
-
-        execute.iinc(metadata.loopCounterSlot, -count);
-        execute.visitVarInsn(Opcodes.ILOAD, metadata.loopCounterSlot);
-        execute.push(0);
-        execute.ifICmp(GeneratorAdapter.GT, end);
-        execute.throwException(PAINLESS_ERROR_TYPE,
-            "The maximum number of statements that can be executed in a loop has been reached.");
-        execute.mark(end);
-    }
-
-    private void writeConstant(final ParserRuleContext source, final Object constant) {
-        if (constant instanceof Number) {
-            writeNumeric(source, constant);
-        } else if (constant instanceof Character) {
-            writeNumeric(source, (int)(char)constant);
-        } else if (constant instanceof String) {
-            writeString(source, constant);
-        } else if (constant instanceof Boolean) {
-            writeBoolean(source, constant);
-        } else if (constant != null) {
-            throw new IllegalStateException(Metadata.error(source) + "Unexpected writer state.");
-        }
-    }
-
-    private void writeNumeric(final ParserRuleContext source, final Object numeric) {
-        if (numeric instanceof Double) {
-            execute.push((double)numeric);
-        } else if (numeric instanceof Float) {
-            execute.push((float)numeric);
-        } else if (numeric instanceof Long) {
-            execute.push((long)numeric);
-        } else if (numeric instanceof Number) {
-            execute.push(((Number)numeric).intValue());
-        } else {
-            throw new IllegalStateException(Metadata.error(source) + "Unexpected writer state.");
-        }
-    }
-
-    private void writeString(final ParserRuleContext source, final Object string) {
-        if (string instanceof String) {
-            execute.push((String)string);
-        } else {
-            throw new IllegalStateException(Metadata.error(source) + "Unexpected writer state.");
-        }
-    }
-
-    private void writeBoolean(final ParserRuleContext source, final Object bool) {
-        if (bool instanceof Boolean) {
-            execute.push((boolean)bool);
-        } else {
-            throw new IllegalStateException(Metadata.error(source) + "Unexpected writer state.");
-        }
-    }
-
-    private void writeNewStrings() {
-        execute.newInstance(STRINGBUILDER_TYPE);
-        execute.dup();
-        execute.invokeConstructor(STRINGBUILDER_TYPE, STRINGBUILDER_CONSTRUCTOR);
-    }
-
-    private void writeAppendStrings(final Sort sort) {
-        switch (sort) {
-            case BOOL:   execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_BOOLEAN); break;
-            case CHAR:   execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_CHAR);    break;
-            case BYTE:
-            case SHORT:
-            case INT:    execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_INT);     break;
-            case LONG:   execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_LONG);    break;
-            case FLOAT:  execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_FLOAT);   break;
-            case DOUBLE: execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_DOUBLE);  break;
-            case STRING: execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_STRING);  break;
-            default:     execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_OBJECT);
-        }
-    }
-
-    private void writeToStrings() {
-        execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_TOSTRING);
-    }
-
-    private void writeBinaryInstruction(final ParserRuleContext source, final Type type, final int token) {
-        final Sort sort = type.sort;
-        final boolean exact = !settings.getNumericOverflow() &&
-            ((sort == Sort.INT || sort == Sort.LONG) &&
-                (token == MUL || token == DIV || token == ADD || token == SUB) ||
-                (sort == Sort.FLOAT || sort == Sort.DOUBLE) &&
-                    (token == MUL || token == DIV || token == REM || token == ADD || token == SUB));
-
-        // if its a 64-bit shift, fixup the lastSource argument to truncate to 32-bits
-        // note unlike java, this means we still do binary promotion of shifts,
-        // but it keeps things simple -- this check works because we promote shifts.
-        if (sort == Sort.LONG && (token == LSH || token == USH || token == RSH)) {
-            execute.cast(org.objectweb.asm.Type.LONG_TYPE, org.objectweb.asm.Type.INT_TYPE);
-        }
-
-        if (exact) {
-            switch (sort) {
-                case INT:
-                    switch (token) {
-                        case MUL: execute.invokeStatic(definition.mathType.type,    MULEXACT_INT);     break;
-                        case DIV: execute.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_INT); break;
-                        case ADD: execute.invokeStatic(definition.mathType.type,    ADDEXACT_INT);     break;
-                        case SUB: execute.invokeStatic(definition.mathType.type,    SUBEXACT_INT);     break;
-                        default:
-                            throw new IllegalStateException(Metadata.error(source) + "Unexpected writer state.");
-                    }
-
-                    break;
-                case LONG:
-                    switch (token) {
-                        case MUL: execute.invokeStatic(definition.mathType.type,    MULEXACT_LONG);     break;
-                        case DIV: execute.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_LONG); break;
-                        case ADD: execute.invokeStatic(definition.mathType.type,    ADDEXACT_LONG);     break;
-                        case SUB: execute.invokeStatic(definition.mathType.type,    SUBEXACT_LONG);     break;
-                        default:
-                            throw new IllegalStateException(Metadata.error(source) + "Unexpected writer state.");
-                    }
-
-                    break;
-                case FLOAT:
-                    switch (token) {
-                        case MUL: execute.invokeStatic(definition.utilityType.type, MULWOOVERLOW_FLOAT); break;
-                        case DIV: execute.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_FLOAT); break;
-                        case REM: execute.invokeStatic(definition.utilityType.type, REMWOOVERLOW_FLOAT); break;
-                        case ADD: execute.invokeStatic(definition.utilityType.type, ADDWOOVERLOW_FLOAT); break;
-                        case SUB: execute.invokeStatic(definition.utilityType.type, SUBWOOVERLOW_FLOAT); break;
-                        default:
-                            throw new IllegalStateException(Metadata.error(source) + "Unexpected writer state.");
-                    }
-
-                    break;
-                case DOUBLE:
-                    switch (token) {
-                        case MUL: execute.invokeStatic(definition.utilityType.type, MULWOOVERLOW_DOUBLE); break;
-                        case DIV: execute.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_DOUBLE); break;
-                        case REM: execute.invokeStatic(definition.utilityType.type, REMWOOVERLOW_DOUBLE); break;
-                        case ADD: execute.invokeStatic(definition.utilityType.type, ADDWOOVERLOW_DOUBLE); break;
-                        case SUB: execute.invokeStatic(definition.utilityType.type, SUBWOOVERLOW_DOUBLE); break;
-                        default:
-                            throw new IllegalStateException(Metadata.error(source) + "Unexpected writer state.");
-                    }
-
-                    break;
-                default:
-                    throw new IllegalStateException(Metadata.error(source) + "Unexpected writer state.");
-            }
-        } else {
-            if ((sort == Sort.FLOAT || sort == Sort.DOUBLE) &&
-                (token == LSH || token == USH || token == RSH || token == BWAND || token == BWXOR || token == BWOR)) {
-                throw new IllegalStateException(Metadata.error(source) + "Unexpected writer state.");
-            }
-
-            if (sort == Sort.DEF) {
-                switch (token) {
-                    case MUL:   execute.invokeStatic(definition.defobjType.type, DEF_MUL_CALL); break;
-                    case DIV:   execute.invokeStatic(definition.defobjType.type, DEF_DIV_CALL); break;
-                    case REM:   execute.invokeStatic(definition.defobjType.type, DEF_REM_CALL); break;
-                    case ADD:   execute.invokeStatic(definition.defobjType.type, DEF_ADD_CALL); break;
-                    case SUB:   execute.invokeStatic(definition.defobjType.type, DEF_SUB_CALL); break;
-                    case LSH:   execute.invokeStatic(definition.defobjType.type, DEF_LSH_CALL); break;
-                    case USH:   execute.invokeStatic(definition.defobjType.type, DEF_RSH_CALL); break;
-                    case RSH:   execute.invokeStatic(definition.defobjType.type, DEF_USH_CALL); break;
-                    case BWAND: execute.invokeStatic(definition.defobjType.type, DEF_AND_CALL); break;
-                    case BWXOR: execute.invokeStatic(definition.defobjType.type, DEF_XOR_CALL); break;
-                    case BWOR:  execute.invokeStatic(definition.defobjType.type, DEF_OR_CALL);  break;
-                    default:
-                        throw new IllegalStateException(Metadata.error(source) + "Unexpected writer state.");
-                }
-            } else {
-                switch (token) {
-                    case MUL:   execute.math(GeneratorAdapter.MUL,  type.type); break;
-                    case DIV:   execute.math(GeneratorAdapter.DIV,  type.type); break;
-                    case REM:   execute.math(GeneratorAdapter.REM,  type.type); break;
-                    case ADD:   execute.math(GeneratorAdapter.ADD,  type.type); break;
-                    case SUB:   execute.math(GeneratorAdapter.SUB,  type.type); break;
-                    case LSH:   execute.math(GeneratorAdapter.SHL,  type.type); break;
-                    case USH:   execute.math(GeneratorAdapter.USHR, type.type); break;
-                    case RSH:   execute.math(GeneratorAdapter.SHR,  type.type); break;
-                    case BWAND: execute.math(GeneratorAdapter.AND,  type.type); break;
-                    case BWXOR: execute.math(GeneratorAdapter.XOR,  type.type); break;
-                    case BWOR:  execute.math(GeneratorAdapter.OR,   type.type); break;
-                    default:
-                        throw new IllegalStateException(Metadata.error(source) + "Unexpected writer state.");
-                }
-            }
-        }
-    }
-
-    /**
-     * Called for any compound assignment (including increment/decrement instructions).
-     * We have to be stricter than writeBinary, and do overflow checks against the original type's size
-     * instead of the promoted type's size, since the result will be implicitly cast back.
-     *
-     * @return true if an instruction is written, false otherwise
-     */
-    private boolean writeExactInstruction(final Sort osort, final Sort psort) {
-        if (psort == Sort.DOUBLE) {
-            if (osort == Sort.FLOAT) {
-                execute.invokeStatic(definition.utilityType.type, TOFLOATWOOVERFLOW_DOUBLE);
-            } else if (osort == Sort.FLOAT_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOFLOATWOOVERFLOW_DOUBLE);
-                execute.checkCast(definition.floatobjType.type);
-            } else if (osort == Sort.LONG) {
-                execute.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_DOUBLE);
-            } else if (osort == Sort.LONG_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_DOUBLE);
-                execute.checkCast(definition.longobjType.type);
-            } else if (osort == Sort.INT) {
-                execute.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_DOUBLE);
-            } else if (osort == Sort.INT_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_DOUBLE);
-                execute.checkCast(definition.intobjType.type);
-            } else if (osort == Sort.CHAR) {
-                execute.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_DOUBLE);
-            } else if (osort == Sort.CHAR_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_DOUBLE);
-                execute.checkCast(definition.charobjType.type);
-            } else if (osort == Sort.SHORT) {
-                execute.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_DOUBLE);
-            } else if (osort == Sort.SHORT_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_DOUBLE);
-                execute.checkCast(definition.shortobjType.type);
-            } else if (osort == Sort.BYTE) {
-                execute.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_DOUBLE);
-            } else if (osort == Sort.BYTE_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_DOUBLE);
-                execute.checkCast(definition.byteobjType.type);
-            } else {
-                return false;
-            }
-        } else if (psort == Sort.FLOAT) {
-            if (osort == Sort.LONG) {
-                execute.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_FLOAT);
-            } else if (osort == Sort.LONG_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_FLOAT);
-                execute.checkCast(definition.longobjType.type);
-            } else if (osort == Sort.INT) {
-                execute.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_FLOAT);
-            } else if (osort == Sort.INT_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_FLOAT);
-                execute.checkCast(definition.intobjType.type);
-            } else if (osort == Sort.CHAR) {
-                execute.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_FLOAT);
-            } else if (osort == Sort.CHAR_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_FLOAT);
-                execute.checkCast(definition.charobjType.type);
-            } else if (osort == Sort.SHORT) {
-                execute.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_FLOAT);
-            } else if (osort == Sort.SHORT_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_FLOAT);
-                execute.checkCast(definition.shortobjType.type);
-            } else if (osort == Sort.BYTE) {
-                execute.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_FLOAT);
-            } else if (osort == Sort.BYTE_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_FLOAT);
-                execute.checkCast(definition.byteobjType.type);
-            } else {
-                return false;
-            }
-        } else if (psort == Sort.LONG) {
-            if (osort == Sort.INT) {
-                execute.invokeStatic(definition.mathType.type, TOINTEXACT_LONG);
-            } else if (osort == Sort.INT_OBJ) {
-                execute.invokeStatic(definition.mathType.type, TOINTEXACT_LONG);
-                execute.checkCast(definition.intobjType.type);
-            } else if (osort == Sort.CHAR) {
-                execute.invokeStatic(definition.utilityType.type, TOCHAREXACT_LONG);
-            } else if (osort == Sort.CHAR_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOCHAREXACT_LONG);
-                execute.checkCast(definition.charobjType.type);
-            } else if (osort == Sort.SHORT) {
-                execute.invokeStatic(definition.utilityType.type, TOSHORTEXACT_LONG);
-            } else if (osort == Sort.SHORT_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOSHORTEXACT_LONG);
-                execute.checkCast(definition.shortobjType.type);
-            } else if (osort == Sort.BYTE) {
-                execute.invokeStatic(definition.utilityType.type, TOBYTEEXACT_LONG);
-            } else if (osort == Sort.BYTE_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOBYTEEXACT_LONG);
-                execute.checkCast(definition.byteobjType.type);
-            } else {
-                return false;
-            }
-        } else if (psort == Sort.INT) {
-            if (osort == Sort.CHAR) {
-                execute.invokeStatic(definition.utilityType.type, TOCHAREXACT_INT);
-            } else if (osort == Sort.CHAR_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOCHAREXACT_INT);
-                execute.checkCast(definition.charobjType.type);
-            } else if (osort == Sort.SHORT) {
-                execute.invokeStatic(definition.utilityType.type, TOSHORTEXACT_INT);
-            } else if (osort == Sort.SHORT_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOSHORTEXACT_INT);
-                execute.checkCast(definition.shortobjType.type);
-            } else if (osort == Sort.BYTE) {
-                execute.invokeStatic(definition.utilityType.type, TOBYTEEXACT_INT);
-            } else if (osort == Sort.BYTE_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOBYTEEXACT_INT);
-                execute.checkCast(definition.byteobjType.type);
-            } else {
-                return false;
-            }
-        } else {
-            return false;
-        }
-
-        return true;
-    }
-
-    private void writeLoadStoreExternal(final ParserRuleContext source) {
-        final Metadata.ExtNodeMetadata sourceenmd = metadata.getExtNodeMetadata(source);
-        final Metadata.ExternalMetadata parentemd = metadata.getExternalMetadata(sourceenmd.parent);
-
-        final boolean length = "#length".equals(sourceenmd.target);
-        final boolean array = "#brace".equals(sourceenmd.target);
-        final boolean name = sourceenmd.target instanceof String && !length && !array;
-        final boolean variable = sourceenmd.target instanceof Integer;
-        final boolean field = sourceenmd.target instanceof Field;
-        final boolean shortcut = sourceenmd.target instanceof Object[];
-
-        if (!length && !variable && !field && !array && !name && !shortcut) {
-            throw new IllegalStateException(Metadata.error(source) + "Target not found for load/store.");
-        }
-
-        final boolean maplist = shortcut && (boolean)((Object[])sourceenmd.target)[2];
-        final Object constant = shortcut ? ((Object[])sourceenmd.target)[3] : null;
-
-        final boolean x1 = field || name || (shortcut && !maplist);
-        final boolean x2 = array || (shortcut && maplist);
-
-        if (length) {
-            execute.arrayLength();
-        } else if (sourceenmd.last && parentemd.storeExpr != null) {
-            final Metadata.ExpressionMetadata expremd = metadata.getExpressionMetadata(parentemd.storeExpr);
-            final boolean cat = strings.contains(parentemd.storeExpr);
-
-            if (cat) {
-                if (field || name || shortcut) {
-                    execute.dupX1();
-                } else if (array) {
-                    execute.dup2X1();
-                }
-
-                if (maplist) {
-                    if (constant != null) {
-                        writeConstant(source, constant);
-                    }
-
-                    execute.dupX2();
-                }
-
-                writeLoadStoreInstruction(source, false, variable, field, name, array, shortcut);
-                writeAppendStrings(sourceenmd.type.sort);
-                visit(parentemd.storeExpr);
-
-                if (strings.contains(parentemd.storeExpr)) {
-                    writeAppendStrings(expremd.to.sort);
-                    strings.remove(parentemd.storeExpr);
-                }
-
-                writeToStrings();
-                checkWriteCast(source, sourceenmd.castTo);
-
-                if (parentemd.read) {
-                    writeDup(sourceenmd.type.sort.size, x1, x2);
-                }
-
-                writeLoadStoreInstruction(source, true, variable, field, name, array, shortcut);
-            } else if (parentemd.token > 0) {
-                final int token = parentemd.token;
-
-                if (field || name || shortcut) {
-                    execute.dup();
-                } else if (array) {
-                    execute.dup2();
-                }
-
-                if (maplist) {
-                    if (constant != null) {
-                        writeConstant(source, constant);
-                    }
-
-                    execute.dupX1();
-                }
-
-                writeLoadStoreInstruction(source, false, variable, field, name, array, shortcut);
-
-                if (parentemd.read && parentemd.post) {
-                    writeDup(sourceenmd.type.sort.size, x1, x2);
-                }
-
-                checkWriteCast(source, sourceenmd.castFrom);
-                visit(parentemd.storeExpr);
-
-                writeBinaryInstruction(source, sourceenmd.promote, token);
-
-                boolean exact = false;
-
-                if (!settings.getNumericOverflow() && expremd.typesafe && sourceenmd.type.sort != Sort.DEF &&
-                    (token == MUL || token == DIV || token == REM || token == ADD || token == SUB)) {
-                    exact = writeExactInstruction(sourceenmd.type.sort, sourceenmd.promote.sort);
-                }
-
-                if (!exact) {
-                    checkWriteCast(source, sourceenmd.castTo);
-                }
-
-                if (parentemd.read && !parentemd.post) {
-                    writeDup(sourceenmd.type.sort.size, x1, x2);
-                }
-
-                writeLoadStoreInstruction(source, true, variable, field, name, array, shortcut);
-            } else {
-                if (constant != null) {
-                    writeConstant(source, constant);
-                }
-
-                visit(parentemd.storeExpr);
-
-                if (parentemd.read) {
-                    writeDup(sourceenmd.type.sort.size, x1, x2);
-                }
-
-                writeLoadStoreInstruction(source, true, variable, field, name, array, shortcut);
-            }
-        } else {
-            if (constant != null) {
-                writeConstant(source, constant);
-            }
-
-            writeLoadStoreInstruction(source, false, variable, field, name, array, shortcut);
-        }
-    }
-
-    private void writeLoadStoreInstruction(final ParserRuleContext source,
-                                           final boolean store, final boolean variable,
-                                           final boolean field, final boolean name,
-                                           final boolean array, final boolean shortcut) {
-        final Metadata.ExtNodeMetadata sourceemd = metadata.getExtNodeMetadata(source);
-
-        if (variable) {
-            writeLoadStoreVariable(source, store, sourceemd.type, (int)sourceemd.target);
-        } else if (field) {
-            writeLoadStoreField(store, (Field)sourceemd.target);
-        } else if (name) {
-            writeLoadStoreField(source, store, (String)sourceemd.target);
-        } else if (array) {
-            writeLoadStoreArray(source, store, sourceemd.type);
-        } else if (shortcut) {
-            Object[] targets = (Object[])sourceemd.target;
-            writeLoadStoreShortcut(store, (Method)targets[0], (Method)targets[1]);
-        } else {
-            throw new IllegalStateException(Metadata.error(source) + "Load/Store requires a variable, field, or array.");
-        }
-    }
-
-    private void writeLoadStoreVariable(final ParserRuleContext source, final boolean store,
-                                        final Type type, final int slot) {
-        if (type.sort == Sort.VOID) {
-            throw new IllegalStateException(Metadata.error(source) + "Cannot load/store void type.");
-        }
-
-        if (store) {
-            execute.visitVarInsn(type.type.getOpcode(Opcodes.ISTORE), slot);
-        } else {
-            execute.visitVarInsn(type.type.getOpcode(Opcodes.ILOAD), slot);
-        }
-    }
-
-    private void writeLoadStoreField(final boolean store, final Field field) {
-        if (java.lang.reflect.Modifier.isStatic(field.reflect.getModifiers())) {
-            if (store) {
-                execute.putStatic(field.owner.type, field.reflect.getName(), field.type.type);
-            } else {
-                execute.getStatic(field.owner.type, field.reflect.getName(), field.type.type);
-
-                if (!field.generic.clazz.equals(field.type.clazz)) {
-                    execute.checkCast(field.generic.type);
-                }
-            }
-        } else {
-            if (store) {
-                execute.putField(field.owner.type, field.reflect.getName(), field.type.type);
-            } else {
-                execute.getField(field.owner.type, field.reflect.getName(), field.type.type);
-
-                if (!field.generic.clazz.equals(field.type.clazz)) {
-                    execute.checkCast(field.generic.type);
-                }
-            }
-        }
-    }
-
-    private void writeLoadStoreField(final ParserRuleContext source, final boolean store, final String name) {
-        if (store) {
-            final Metadata.ExtNodeMetadata sourceemd = metadata.getExtNodeMetadata(source);
-            final Metadata.ExternalMetadata parentemd = metadata.getExternalMetadata(sourceemd.parent);
-            final Metadata.ExpressionMetadata expremd = metadata.getExpressionMetadata(parentemd.storeExpr);
-
-            execute.push(name);
-            execute.loadThis();
-            execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE);
-            execute.push(parentemd.token == 0 && expremd.typesafe);
-            execute.invokeStatic(definition.defobjType.type, DEF_FIELD_STORE);
-        } else {
-            execute.push(name);
-            execute.loadThis();
-            execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE);
-            execute.invokeStatic(definition.defobjType.type, DEF_FIELD_LOAD);
-        }
-    }
-
-    private void writeLoadStoreArray(final ParserRuleContext source, final boolean store, final Type type) {
-        if (type.sort == Sort.VOID) {
-            throw new IllegalStateException(Metadata.error(source) + "Cannot load/store void type.");
-        }
-
-        if (type.sort == Sort.DEF) {
-            final ExtbraceContext bracectx = (ExtbraceContext)source;
-            final Metadata.ExpressionMetadata expremd0 = metadata.getExpressionMetadata(bracectx.expression());
-
-            if (store) {
-                final Metadata.ExtNodeMetadata braceenmd = metadata.getExtNodeMetadata(bracectx);
-                final Metadata.ExternalMetadata parentemd = metadata.getExternalMetadata(braceenmd.parent);
-                final Metadata.ExpressionMetadata expremd1 = metadata.getExpressionMetadata(parentemd.storeExpr);
-
-                execute.loadThis();
-                execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE);
-                execute.push(expremd0.typesafe);
-                execute.push(parentemd.token == 0 && expremd1.typesafe);
-                execute.invokeStatic(definition.defobjType.type, DEF_ARRAY_STORE);
-            } else {
-                execute.loadThis();
-                execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE);
-                execute.push(expremd0.typesafe);
-                execute.invokeStatic(definition.defobjType.type, DEF_ARRAY_LOAD);
-            }
-        } else {
-            if (store) {
-                execute.arrayStore(type.type);
-            } else {
-                execute.arrayLoad(type.type);
-            }
-        }
-    }
-
-    private void writeLoadStoreShortcut(final boolean store, final Method getter, final Method setter) {
-        final Method method = store ? setter : getter;
-
-        if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
-            execute.invokeInterface(method.owner.type, method.method);
-        } else {
-            execute.invokeVirtual(method.owner.type, method.method);
-        }
-
-        if (store) {
-            writePop(method.rtn.type.getSize());
-        } else if (!method.rtn.clazz.equals(method.handle.type().returnType())) {
-            execute.checkCast(method.rtn.type);
-        }
-    }
-
-    private void writeDup(final int size, final boolean x1, final boolean x2) {
-        if (size == 1) {
-            if (x2) {
-                execute.dupX2();
-            } else if (x1) {
-                execute.dupX1();
-            } else {
-                execute.dup();
-            }
-        } else if (size == 2) {
-            if (x2) {
-                execute.dup2X2();
-            } else if (x1) {
-                execute.dup2X1();
-            } else {
-                execute.dup2();
-            }
-        }
-    }
-
-    private void writeNewExternal(final ExtnewContext source) {
-        final Metadata.ExtNodeMetadata sourceenmd = metadata.getExtNodeMetadata(source);
-        final Metadata.ExternalMetadata parentemd = metadata.getExternalMetadata(sourceenmd.parent);
-
-        final boolean makearray = "#makearray".equals(sourceenmd.target);
-        final boolean constructor = sourceenmd.target instanceof Constructor;
-
-        if (!makearray && !constructor) {
-            throw new IllegalStateException(Metadata.error(source) + "Target not found for new call.");
-        }
-
-        if (makearray) {
-            for (final ExpressionContext exprctx : source.expression()) {
-                visit(exprctx);
-            }
-
-            if (sourceenmd.type.sort == Sort.ARRAY) {
-                execute.visitMultiANewArrayInsn(sourceenmd.type.type.getDescriptor(), sourceenmd.type.type.getDimensions());
-            } else {
-                execute.newArray(sourceenmd.type.type);
-            }
-        } else {
-            execute.newInstance(sourceenmd.type.type);
-
-            if (parentemd.read) {
-                execute.dup();
-            }
-
-            for (final ExpressionContext exprctx : source.arguments().expression()) {
-                visit(exprctx);
-            }
-
-            final Constructor target = (Constructor)sourceenmd.target;
-            execute.invokeConstructor(target.owner.type, target.method);
-        }
-    }
-
-    private void writeCallExternal(final ExtcallContext source) {
-        final Metadata.ExtNodeMetadata sourceenmd = metadata.getExtNodeMetadata(source);
-
-        final boolean method = sourceenmd.target instanceof Method;
-        final boolean def = sourceenmd.target instanceof String;
-
-        if (!method && !def) {
-            throw new IllegalStateException(Metadata.error(source) + "Target not found for call.");
-        }
-
-        final List<ExpressionContext> arguments = source.arguments().expression();
-
-        if (method) {
-            for (final ExpressionContext exprctx : arguments) {
-                visit(exprctx);
-            }
-
-            final Method target = (Method)sourceenmd.target;
-
-            if (java.lang.reflect.Modifier.isStatic(target.reflect.getModifiers())) {
-                execute.invokeStatic(target.owner.type, target.method);
-            } else if (java.lang.reflect.Modifier.isInterface(target.owner.clazz.getModifiers())) {
-                execute.invokeInterface(target.owner.type, target.method);
-            } else {
-                execute.invokeVirtual(target.owner.type, target.method);
-            }
-
-            if (!target.rtn.clazz.equals(target.handle.type().returnType())) {
-                execute.checkCast(target.rtn.type);
-            }
-        } else {
-            execute.push((String)sourceenmd.target);
-            execute.loadThis();
-            execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE);
-
-            execute.push(arguments.size());
-            execute.newArray(definition.defType.type);
-
-            for (int argument = 0; argument < arguments.size(); ++argument) {
-                execute.dup();
-                execute.push(argument);
-                visit(arguments.get(argument));
-                execute.arrayStore(definition.defType.type);
-            }
-
-            execute.push(arguments.size());
-            execute.newArray(definition.booleanType.type);
-
-            for (int argument = 0; argument < arguments.size(); ++argument) {
-                execute.dup();
-                execute.push(argument);
-                execute.push(metadata.getExpressionMetadata(arguments.get(argument)).typesafe);
-                execute.arrayStore(definition.booleanType.type);
-            }
-
-            execute.invokeStatic(definition.defobjType.type, DEF_METHOD_CALL);
-        }
-    }
-
-    private void writePop(final int size) {
-        if (size == 1) {
-            execute.pop();
-        } else if (size == 2) {
-            execute.pop2();
-        }
-    }
-
-    private void checkWriteCast(final Metadata.ExpressionMetadata sort) {
-        checkWriteCast(sort.source, sort.cast);
-    }
-
-    private void checkWriteCast(final ParserRuleContext source, final Cast cast) {
-        if (cast instanceof Transform) {
-            writeTransform((Transform)cast);
-        } else if (cast != null) {
-            writeCast(cast);
-        } else {
-            throw new IllegalStateException(Metadata.error(source) + "Unexpected cast object.");
-        }
-    }
-
-    private void writeCast(final Cast cast) {
-        final Type from = cast.from;
-        final Type to = cast.to;
-
-        if (from.equals(to)) {
-            return;
-        }
-
-        if (from.sort.numeric && from.sort.primitive && to.sort.numeric && to.sort.primitive) {
-            execute.cast(from.type, to.type);
-        } else {
-            try {
-                from.clazz.asSubclass(to.clazz);
-            } catch (ClassCastException exception) {
-                execute.checkCast(to.type);
-            }
-        }
-    }
-
-    private void writeTransform(final Transform transform) {
-        if (transform.upcast != null) {
-            execute.checkCast(transform.upcast.type);
-        }
-
-        if (java.lang.reflect.Modifier.isStatic(transform.method.reflect.getModifiers())) {
-            execute.invokeStatic(transform.method.owner.type, transform.method.method);
-        } else if (java.lang.reflect.Modifier.isInterface(transform.method.owner.clazz.getModifiers())) {
-            execute.invokeInterface(transform.method.owner.type, transform.method.method);
-        } else {
-            execute.invokeVirtual(transform.method.owner.type, transform.method.method);
-        }
-
-        if (transform.downcast != null) {
-            execute.checkCast(transform.downcast.type);
-        }
-    }
-
-    void checkWriteBranch(final ParserRuleContext source) {
-        final Branch branch = getBranch(source);
-
-        if (branch != null) {
-            if (branch.tru != null) {
-                execute.visitJumpInsn(Opcodes.IFNE, branch.tru);
-            } else if (branch.fals != null) {
-                execute.visitJumpInsn(Opcodes.IFEQ, branch.fals);
-            }
-        }
-    }
-
     private void writeEnd() {
         writer.visitEnd();
     }
@@ -2376,4 +179,352 @@ class Writer extends PainlessParserBaseVisitor<Void> {
     private byte[] getBytes() {
         return writer.toByteArray();
     }
+
+    @Override
+    public Void visitSource(final SourceContext ctx) {
+        statement.processSource(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitIf(final IfContext ctx) {
+        statement.processIf(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitWhile(final WhileContext ctx) {
+        statement.processWhile(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitDo(final DoContext ctx) {
+        statement.processDo(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitFor(final ForContext ctx) {
+        statement.processFor(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitDecl(final DeclContext ctx) {
+        statement.processDecl(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitContinue(final ContinueContext ctx) {
+        statement.processContinue();
+
+        return null;
+    }
+
+    @Override
+    public Void visitBreak(final BreakContext ctx) {
+        statement.processBreak();
+
+        return null;
+    }
+
+    @Override
+    public Void visitReturn(final ReturnContext ctx) {
+        statement.processReturn(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitTry(final TryContext ctx) {
+        statement.processTry(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitThrow(final ThrowContext ctx) {
+        statement.processThrow(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitExpr(final ExprContext ctx) {
+        statement.processExpr(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitMultiple(final MultipleContext ctx) {
+        statement.processMultiple(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitSingle(final SingleContext ctx) {
+        statement.processSingle(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitEmpty(final EmptyContext ctx) {
+        throw new UnsupportedOperationException(WriterUtility.error(ctx) + "Unexpected state.");
+    }
+
+    @Override
+    public Void visitEmptyscope(final EmptyscopeContext ctx) {
+        throw new UnsupportedOperationException(WriterUtility.error(ctx) + "Unexpected state.");
+    }
+
+    @Override
+    public Void visitInitializer(final InitializerContext ctx) {
+        statement.processInitializer(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitAfterthought(final AfterthoughtContext ctx) {
+        statement.processAfterthought(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitDeclaration(DeclarationContext ctx) {
+        statement.processDeclaration(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitDecltype(final DecltypeContext ctx) {
+        throw new UnsupportedOperationException(WriterUtility.error(ctx) + "Unexpected state.");
+    }
+
+    @Override
+    public Void visitDeclvar(final DeclvarContext ctx) {
+        statement.processDeclvar(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitTrap(final TrapContext ctx) {
+        statement.processTrap(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitPrecedence(final PrecedenceContext ctx) {
+        throw new UnsupportedOperationException(WriterUtility.error(ctx) + "Unexpected state.");
+    }
+
+    @Override
+    public Void visitNumeric(final NumericContext ctx) {
+        expression.processNumeric(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitChar(final CharContext ctx) {
+        expression.processChar(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitTrue(final TrueContext ctx) {
+        expression.processTrue(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitFalse(final FalseContext ctx) {
+        expression.processFalse(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitNull(final NullContext ctx) {
+        expression.processNull(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitExternal(final ExternalContext ctx) {
+        expression.processExternal(ctx);
+
+        return null;
+    }
+
+
+    @Override
+    public Void visitPostinc(final PostincContext ctx) {
+        expression.processPostinc(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitPreinc(final PreincContext ctx) {
+        expression.processPreinc(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitUnary(final UnaryContext ctx) {
+        expression.processUnary(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitCast(final CastContext ctx) {
+        expression.processCast(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitBinary(final BinaryContext ctx) {
+        expression.processBinary(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitComp(final CompContext ctx) {
+        expression.processComp(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitBool(final BoolContext ctx) {
+        expression.processBool(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitConditional(final ConditionalContext ctx) {
+        expression.processConditional(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitAssignment(final AssignmentContext ctx) {
+        expression.processAssignment(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitExtstart(final ExtstartContext ctx) {
+        external.processExtstart(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitExtprec(final ExtprecContext ctx) {
+        external.processExtprec(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitExtcast(final ExtcastContext ctx) {
+        external.processExtcast(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitExtbrace(final ExtbraceContext ctx) {
+        external.processExtbrace(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitExtdot(final ExtdotContext ctx) {
+        external.processExtdot(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitExttype(final ExttypeContext ctx) {
+        external.processExttype(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitExtcall(final ExtcallContext ctx) {
+        external.processExtcall(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitExtvar(final ExtvarContext ctx) {
+        external.processExtvar(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitExtfield(final ExtfieldContext ctx) {
+        external.processExtfield(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitExtnew(final ExtnewContext ctx) {
+        external.processExtnew(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitExtstring(final ExtstringContext ctx) {
+        external.processExtstring(ctx);
+
+        return null;
+    }
+
+    @Override
+    public Void visitArguments(final ArgumentsContext ctx) {
+        throw new UnsupportedOperationException(WriterUtility.error(ctx) + "Unexpected state.");
+    }
+
+    @Override
+    public Void visitIncrement(final IncrementContext ctx) {
+        expression.processIncrement(ctx);
+
+        return null;
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterCaster.java
new file mode 100644
index 00000000000..c55dff2f549
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterCaster.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.elasticsearch.painless.Definition.Cast;
+import org.elasticsearch.painless.Definition.Transform;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Metadata.ExpressionMetadata;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+class WriterCaster {
+    private final GeneratorAdapter execute;
+
+    WriterCaster(final GeneratorAdapter execute) {
+        this.execute = execute;
+    }
+
+    void checkWriteCast(final ExpressionMetadata sort) {
+        checkWriteCast(sort.source, sort.cast);
+    }
+
+    void checkWriteCast(final ParserRuleContext source, final Cast cast) {
+        if (cast instanceof Transform) {
+            writeTransform((Transform)cast);
+        } else if (cast != null) {
+            writeCast(cast);
+        } else {
+            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected cast object.");
+        }
+    }
+
+    private void writeCast(final Cast cast) {
+        final Type from = cast.from;
+        final Type to = cast.to;
+
+        if (from.equals(to)) {
+            return;
+        }
+
+        if (from.sort.numeric && from.sort.primitive && to.sort.numeric && to.sort.primitive) {
+            execute.cast(from.type, to.type);
+        } else {
+            try {
+                from.clazz.asSubclass(to.clazz);
+            } catch (ClassCastException exception) {
+                execute.checkCast(to.type);
+            }
+        }
+    }
+
+    private void writeTransform(final Transform transform) {
+        if (transform.upcast != null) {
+            execute.checkCast(transform.upcast.type);
+        }
+
+        if (java.lang.reflect.Modifier.isStatic(transform.method.reflect.getModifiers())) {
+            execute.invokeStatic(transform.method.owner.type, transform.method.method);
+        } else if (java.lang.reflect.Modifier.isInterface(transform.method.owner.clazz.getModifiers())) {
+            execute.invokeInterface(transform.method.owner.type, transform.method.method);
+        } else {
+            execute.invokeVirtual(transform.method.owner.type, transform.method.method);
+        }
+
+        if (transform.downcast != null) {
+            execute.checkCast(transform.downcast.type);
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
new file mode 100644
index 00000000000..33fea094058
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.script.ScoreAccessor;
+import org.objectweb.asm.Type;
+import org.objectweb.asm.commons.Method;
+
+import java.lang.invoke.MethodType;
+import java.util.Map;
+
+class WriterConstants {
+    final static String BASE_CLASS_NAME = Executable.class.getName();
+    final static String CLASS_NAME      = BASE_CLASS_NAME + "$CompiledPainlessExecutable";
+    final static Type BASE_CLASS_TYPE   = Type.getType(Executable.class);
+    final static Type CLASS_TYPE        = Type.getType("L" + CLASS_NAME.replace(".", "/") + ";");
+
+    final static Method CONSTRUCTOR = getAsmMethod(void.class, "<init>", Definition.class, String.class, String.class);
+    final static Method EXECUTE     = getAsmMethod(Object.class, "execute", Map.class);
+    final static String SIGNATURE   = "(Ljava/util/Map<Ljava/lang/String;Ljava/lang/Object;>;)Ljava/lang/Object;";
+
+    final static Type PAINLESS_ERROR_TYPE = Type.getType(PainlessError.class);
+
+    final static Type DEFINITION_TYPE = Type.getType(Definition.class);
+
+    final static Type MAP_TYPE  = Type.getType(Map.class);
+    final static Method MAP_GET = getAsmMethod(Object.class, "get", Object.class);
+
+    final static Type SCORE_ACCESSOR_TYPE    = Type.getType(ScoreAccessor.class);
+    final static Method SCORE_ACCESSOR_FLOAT = getAsmMethod(float.class, "floatValue");
+
+    final static Method DEF_METHOD_CALL = getAsmMethod(
+        Object.class, "methodCall", Object.class, String.class, Definition.class, Object[].class, boolean[].class);
+    final static Method DEF_ARRAY_STORE = getAsmMethod(
+        void.class, "arrayStore", Object.class, Object.class, Object.class, Definition.class, boolean.class, boolean.class);
+    final static Method DEF_ARRAY_LOAD = getAsmMethod(
+        Object.class, "arrayLoad", Object.class, Object.class, Definition.class, boolean.class);
+    final static Method DEF_FIELD_STORE = getAsmMethod(
+        void.class, "fieldStore", Object.class, Object.class, String.class, Definition.class, boolean.class);
+    final static Method DEF_FIELD_LOAD = getAsmMethod(
+        Object.class, "fieldLoad", Object.class, String.class, Definition.class);
+
+    final static Method DEF_NOT_CALL = getAsmMethod(Object.class, "not", Object.class);
+    final static Method DEF_NEG_CALL = getAsmMethod(Object.class, "neg", Object.class);
+    final static Method DEF_MUL_CALL = getAsmMethod(Object.class, "mul", Object.class, Object.class);
+    final static Method DEF_DIV_CALL = getAsmMethod(Object.class, "div", Object.class, Object.class);
+    final static Method DEF_REM_CALL = getAsmMethod(Object.class, "rem", Object.class, Object.class);
+    final static Method DEF_ADD_CALL = getAsmMethod(Object.class, "add", Object.class, Object.class);
+    final static Method DEF_SUB_CALL = getAsmMethod(Object.class, "sub", Object.class, Object.class);
+    final static Method DEF_LSH_CALL = getAsmMethod(Object.class, "lsh", Object.class, Object.class);
+    final static Method DEF_RSH_CALL = getAsmMethod(Object.class, "rsh", Object.class, Object.class);
+    final static Method DEF_USH_CALL = getAsmMethod(Object.class, "ush", Object.class, Object.class);
+    final static Method DEF_AND_CALL = getAsmMethod(Object.class, "and", Object.class, Object.class);
+    final static Method DEF_XOR_CALL = getAsmMethod(Object.class, "xor", Object.class, Object.class);
+    final static Method DEF_OR_CALL  = getAsmMethod(Object.class, "or" , Object.class, Object.class);
+    final static Method DEF_EQ_CALL  = getAsmMethod(boolean.class, "eq" , Object.class, Object.class);
+    final static Method DEF_LT_CALL  = getAsmMethod(boolean.class, "lt" , Object.class, Object.class);
+    final static Method DEF_LTE_CALL = getAsmMethod(boolean.class, "lte", Object.class, Object.class);
+    final static Method DEF_GT_CALL  = getAsmMethod(boolean.class, "gt" , Object.class, Object.class);
+    final static Method DEF_GTE_CALL = getAsmMethod(boolean.class, "gte", Object.class, Object.class);
+
+    final static Type STRINGBUILDER_TYPE = Type.getType(StringBuilder.class);
+
+    final static Method STRINGBUILDER_CONSTRUCTOR    = getAsmMethod(void.class, "<init>");
+    final static Method STRINGBUILDER_APPEND_BOOLEAN = getAsmMethod(StringBuilder.class, "append", boolean.class);
+    final static Method STRINGBUILDER_APPEND_CHAR    = getAsmMethod(StringBuilder.class, "append", char.class);
+    final static Method STRINGBUILDER_APPEND_INT     = getAsmMethod(StringBuilder.class, "append", int.class);
+    final static Method STRINGBUILDER_APPEND_LONG    = getAsmMethod(StringBuilder.class, "append", long.class);
+    final static Method STRINGBUILDER_APPEND_FLOAT   = getAsmMethod(StringBuilder.class, "append", float.class);
+    final static Method STRINGBUILDER_APPEND_DOUBLE  = getAsmMethod(StringBuilder.class, "append", double.class);
+    final static Method STRINGBUILDER_APPEND_STRING  = getAsmMethod(StringBuilder.class, "append", String.class);
+    final static Method STRINGBUILDER_APPEND_OBJECT  = getAsmMethod(StringBuilder.class, "append", Object.class);
+    final static Method STRINGBUILDER_TOSTRING       = getAsmMethod(String.class, "toString");
+
+    final static Method TOINTEXACT_LONG  = getAsmMethod(int.class,  "toIntExact",    long.class);
+    final static Method NEGATEEXACT_INT  = getAsmMethod(int.class,  "negateExact",   int.class);
+    final static Method NEGATEEXACT_LONG = getAsmMethod(long.class, "negateExact",   long.class);
+    final static Method MULEXACT_INT     = getAsmMethod(int.class,  "multiplyExact", int.class,  int.class);
+    final static Method MULEXACT_LONG    = getAsmMethod(long.class, "multiplyExact", long.class, long.class);
+    final static Method ADDEXACT_INT     = getAsmMethod(int.class,  "addExact",      int.class,  int.class);
+    final static Method ADDEXACT_LONG    = getAsmMethod(long.class, "addExact",      long.class, long.class);
+    final static Method SUBEXACT_INT     = getAsmMethod(int.class,  "subtractExact", int.class,  int.class);
+    final static Method SUBEXACT_LONG    = getAsmMethod(long.class, "subtractExact", long.class, long.class);
+
+    final static Method CHECKEQUALS              = getAsmMethod(boolean.class, "checkEquals",              Object.class, Object.class);
+    final static Method TOBYTEEXACT_INT          = getAsmMethod(byte.class,    "toByteExact",              int.class);
+    final static Method TOBYTEEXACT_LONG         = getAsmMethod(byte.class,    "toByteExact",              long.class);
+    final static Method TOBYTEWOOVERFLOW_FLOAT   = getAsmMethod(byte.class,    "toByteWithoutOverflow",    float.class);
+    final static Method TOBYTEWOOVERFLOW_DOUBLE  = getAsmMethod(byte.class,    "toByteWithoutOverflow",    double.class);
+    final static Method TOSHORTEXACT_INT         = getAsmMethod(short.class,   "toShortExact",             int.class);
+    final static Method TOSHORTEXACT_LONG        = getAsmMethod(short.class,   "toShortExact",             long.class);
+    final static Method TOSHORTWOOVERFLOW_FLOAT  = getAsmMethod(short.class,   "toShortWithoutOverflow",   float.class);
+    final static Method TOSHORTWOOVERFLOW_DOUBLE = getAsmMethod(short.class,   "toShortWihtoutOverflow",   double.class);
+    final static Method TOCHAREXACT_INT          = getAsmMethod(char.class,    "toCharExact",              int.class);
+    final static Method TOCHAREXACT_LONG         = getAsmMethod(char.class,    "toCharExact",              long.class);
+    final static Method TOCHARWOOVERFLOW_FLOAT   = getAsmMethod(char.class,    "toCharWithoutOverflow",    float.class);
+    final static Method TOCHARWOOVERFLOW_DOUBLE  = getAsmMethod(char.class,    "toCharWithoutOverflow",    double.class);
+    final static Method TOINTWOOVERFLOW_FLOAT    = getAsmMethod(int.class,     "toIntWithoutOverflow",     float.class);
+    final static Method TOINTWOOVERFLOW_DOUBLE   = getAsmMethod(int.class,     "toIntWithoutOverflow",     double.class);
+    final static Method TOLONGWOOVERFLOW_FLOAT   = getAsmMethod(long.class,    "toLongWithoutOverflow",    float.class);
+    final static Method TOLONGWOOVERFLOW_DOUBLE  = getAsmMethod(long.class,    "toLongWithoutOverflow",    double.class);
+    final static Method TOFLOATWOOVERFLOW_DOUBLE = getAsmMethod(float.class ,  "toFloatWihtoutOverflow",   double.class);
+    final static Method MULWOOVERLOW_FLOAT       = getAsmMethod(float.class,   "multiplyWithoutOverflow",  float.class,  float.class);
+    final static Method MULWOOVERLOW_DOUBLE      = getAsmMethod(double.class,  "multiplyWithoutOverflow",  double.class, double.class);
+    final static Method DIVWOOVERLOW_INT         = getAsmMethod(int.class,     "divideWithoutOverflow",    int.class,    int.class);
+    final static Method DIVWOOVERLOW_LONG        = getAsmMethod(long.class,    "divideWithoutOverflow",    long.class,   long.class);
+    final static Method DIVWOOVERLOW_FLOAT       = getAsmMethod(float.class,   "divideWithoutOverflow",    float.class,  float.class);
+    final static Method DIVWOOVERLOW_DOUBLE      = getAsmMethod(double.class,  "divideWithoutOverflow",    double.class, double.class);
+    final static Method REMWOOVERLOW_FLOAT       = getAsmMethod(float.class,   "remainderWithoutOverflow", float.class,  float.class);
+    final static Method REMWOOVERLOW_DOUBLE      = getAsmMethod(double.class,  "remainderWithoutOverflow", double.class, double.class);
+    final static Method ADDWOOVERLOW_FLOAT       = getAsmMethod(float.class,   "addWithoutOverflow",       float.class,  float.class);
+    final static Method ADDWOOVERLOW_DOUBLE      = getAsmMethod(double.class,  "addWithoutOverflow",       double.class, double.class);
+    final static Method SUBWOOVERLOW_FLOAT       = getAsmMethod(float.class,   "subtractWithoutOverflow",  float.class,  float.class);
+    final static Method SUBWOOVERLOW_DOUBLE      = getAsmMethod(double.class,  "subtractWithoutOverflow",  double.class, double.class);
+
+    private static Method getAsmMethod(final Class<?> rtype, final String name, final Class<?>... ptypes) {
+        return new Method(name, MethodType.methodType(rtype, ptypes).toMethodDescriptorString());
+    }
+
+    private WriterConstants() {}
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExpression.java
new file mode 100644
index 00000000000..c850031efa0
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExpression.java
@@ -0,0 +1,684 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Metadata.ExpressionMetadata;
+import org.elasticsearch.painless.PainlessParser.AssignmentContext;
+import org.elasticsearch.painless.PainlessParser.BinaryContext;
+import org.elasticsearch.painless.PainlessParser.BoolContext;
+import org.elasticsearch.painless.PainlessParser.CastContext;
+import org.elasticsearch.painless.PainlessParser.CharContext;
+import org.elasticsearch.painless.PainlessParser.CompContext;
+import org.elasticsearch.painless.PainlessParser.ConditionalContext;
+import org.elasticsearch.painless.PainlessParser.ExpressionContext;
+import org.elasticsearch.painless.PainlessParser.ExternalContext;
+import org.elasticsearch.painless.PainlessParser.FalseContext;
+import org.elasticsearch.painless.PainlessParser.IncrementContext;
+import org.elasticsearch.painless.PainlessParser.NullContext;
+import org.elasticsearch.painless.PainlessParser.NumericContext;
+import org.elasticsearch.painless.PainlessParser.PostincContext;
+import org.elasticsearch.painless.PainlessParser.PreincContext;
+import org.elasticsearch.painless.PainlessParser.TrueContext;
+import org.elasticsearch.painless.PainlessParser.UnaryContext;
+import org.elasticsearch.painless.WriterUtility.Branch;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import static org.elasticsearch.painless.PainlessParser.ADD;
+import static org.elasticsearch.painless.PainlessParser.BWAND;
+import static org.elasticsearch.painless.PainlessParser.BWOR;
+import static org.elasticsearch.painless.PainlessParser.BWXOR;
+import static org.elasticsearch.painless.PainlessParser.DIV;
+import static org.elasticsearch.painless.PainlessParser.LSH;
+import static org.elasticsearch.painless.PainlessParser.MUL;
+import static org.elasticsearch.painless.PainlessParser.REM;
+import static org.elasticsearch.painless.PainlessParser.RSH;
+import static org.elasticsearch.painless.PainlessParser.SUB;
+import static org.elasticsearch.painless.PainlessParser.USH;
+import static org.elasticsearch.painless.WriterConstants.CHECKEQUALS;
+import static org.elasticsearch.painless.WriterConstants.DEF_EQ_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_GTE_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_GT_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_LTE_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_LT_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_NEG_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_NOT_CALL;
+import static org.elasticsearch.painless.WriterConstants.NEGATEEXACT_INT;
+import static org.elasticsearch.painless.WriterConstants.NEGATEEXACT_LONG;
+
+class WriterExpression {
+    private final Metadata metadata;
+    private final Definition definition;
+    private final CompilerSettings settings;
+
+    private final GeneratorAdapter execute;
+
+    private final Writer writer;
+    private final WriterUtility utility;
+    private final WriterCaster caster;
+
+    WriterExpression(final Metadata metadata, final GeneratorAdapter execute, final Writer writer,
+                     final WriterUtility utility, final WriterCaster caster) {
+        this.metadata = metadata;
+        definition = metadata.definition;
+        settings = metadata.settings;
+
+        this.execute = execute;
+
+        this.writer = writer;
+        this.utility = utility;
+        this.caster = caster;
+    }
+
+    void processNumeric(final NumericContext ctx) {
+        final ExpressionMetadata numericemd = metadata.getExpressionMetadata(ctx);
+        final Object postConst = numericemd.postConst;
+
+        if (postConst == null) {
+            utility.writeNumeric(ctx, numericemd.preConst);
+            caster.checkWriteCast(numericemd);
+        } else {
+            utility.writeConstant(ctx, postConst);
+        }
+
+        utility.checkWriteBranch(ctx);
+    }
+
+    void processChar(final CharContext ctx) {
+        final ExpressionMetadata charemd = metadata.getExpressionMetadata(ctx);
+        final Object postConst = charemd.postConst;
+
+        if (postConst == null) {
+            utility.writeNumeric(ctx, (int)(char)charemd.preConst);
+            caster.checkWriteCast(charemd);
+        } else {
+            utility.writeConstant(ctx, postConst);
+        }
+
+        utility.checkWriteBranch(ctx);
+    }
+
+    void processTrue(final TrueContext ctx) {
+        final ExpressionMetadata trueemd = metadata.getExpressionMetadata(ctx);
+        final Object postConst = trueemd.postConst;
+        final Branch branch = utility.getBranch(ctx);
+
+        if (branch == null) {
+            if (postConst == null) {
+                utility.writeBoolean(ctx, true);
+                caster.checkWriteCast(trueemd);
+            } else {
+                utility.writeConstant(ctx, postConst);
+            }
+        } else if (branch.tru != null) {
+            execute.goTo(branch.tru);
+        }
+    }
+
+    void processFalse(final FalseContext ctx) {
+        final ExpressionMetadata falseemd = metadata.getExpressionMetadata(ctx);
+        final Object postConst = falseemd.postConst;
+        final Branch branch = utility.getBranch(ctx);
+
+        if (branch == null) {
+            if (postConst == null) {
+                utility.writeBoolean(ctx, false);
+                caster.checkWriteCast(falseemd);
+            } else {
+                utility.writeConstant(ctx, postConst);
+            }
+        } else if (branch.fals != null) {
+            execute.goTo(branch.fals);
+        }
+    }
+
+    void processNull(final NullContext ctx) {
+        final ExpressionMetadata nullemd = metadata.getExpressionMetadata(ctx);
+
+        execute.visitInsn(Opcodes.ACONST_NULL);
+        caster.checkWriteCast(nullemd);
+        utility.checkWriteBranch(ctx);
+    }
+
+    void processExternal(final ExternalContext ctx) {
+        final ExpressionMetadata expremd = metadata.getExpressionMetadata(ctx);
+        writer.visit(ctx.extstart());
+        caster.checkWriteCast(expremd);
+        utility.checkWriteBranch(ctx);
+    }
+
+
+    void processPostinc(final PostincContext ctx) {
+        final ExpressionMetadata expremd = metadata.getExpressionMetadata(ctx);
+        writer.visit(ctx.extstart());
+        caster.checkWriteCast(expremd);
+        utility.checkWriteBranch(ctx);
+    }
+
+    void processPreinc(final PreincContext ctx) {
+        final ExpressionMetadata expremd = metadata.getExpressionMetadata(ctx);
+        writer.visit(ctx.extstart());
+        caster.checkWriteCast(expremd);
+        utility.checkWriteBranch(ctx);
+    }
+
+    void processUnary(final UnaryContext ctx) {
+        final ExpressionMetadata unaryemd = metadata.getExpressionMetadata(ctx);
+        final Object postConst = unaryemd.postConst;
+        final Object preConst = unaryemd.preConst;
+        final Branch branch = utility.getBranch(ctx);
+
+        if (postConst != null) {
+            if (ctx.BOOLNOT() != null) {
+                if (branch == null) {
+                    utility.writeConstant(ctx, postConst);
+                } else {
+                    if ((boolean)postConst && branch.tru != null) {
+                        execute.goTo(branch.tru);
+                    } else if (!(boolean)postConst && branch.fals != null) {
+                        execute.goTo(branch.fals);
+                    }
+                }
+            } else {
+                utility.writeConstant(ctx, postConst);
+                utility.checkWriteBranch(ctx);
+            }
+        } else if (preConst != null) {
+            if (branch == null) {
+                utility.writeConstant(ctx, preConst);
+                caster.checkWriteCast(unaryemd);
+            } else {
+                throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+            }
+        } else {
+            final ExpressionContext exprctx = ctx.expression();
+
+            if (ctx.BOOLNOT() != null) {
+                final Branch local = utility.markBranch(ctx, exprctx);
+
+                if (branch == null) {
+                    local.fals = new Label();
+                    final Label aend = new Label();
+
+                    writer.visit(exprctx);
+
+                    execute.push(false);
+                    execute.goTo(aend);
+                    execute.mark(local.fals);
+                    execute.push(true);
+                    execute.mark(aend);
+
+                    caster.checkWriteCast(unaryemd);
+                } else {
+                    local.tru = branch.fals;
+                    local.fals = branch.tru;
+
+                    writer.visit(exprctx);
+                }
+            } else {
+                final org.objectweb.asm.Type type = unaryemd.from.type;
+                final Sort sort = unaryemd.from.sort;
+
+                writer.visit(exprctx);
+
+                if (ctx.BWNOT() != null) {
+                    if (sort == Sort.DEF) {
+                        execute.invokeStatic(definition.defobjType.type, DEF_NOT_CALL);
+                    } else {
+                        if (sort == Sort.INT) {
+                            utility.writeConstant(ctx, -1);
+                        } else if (sort == Sort.LONG) {
+                            utility.writeConstant(ctx, -1L);
+                        } else {
+                            throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+                        }
+
+                        execute.math(GeneratorAdapter.XOR, type);
+                    }
+                } else if (ctx.SUB() != null) {
+                    if (sort == Sort.DEF) {
+                        execute.invokeStatic(definition.defobjType.type, DEF_NEG_CALL);
+                    } else {
+                        if (settings.getNumericOverflow()) {
+                            execute.math(GeneratorAdapter.NEG, type);
+                        } else {
+                            if (sort == Sort.INT) {
+                                execute.invokeStatic(definition.mathType.type, NEGATEEXACT_INT);
+                            } else if (sort == Sort.LONG) {
+                                execute.invokeStatic(definition.mathType.type, NEGATEEXACT_LONG);
+                            } else {
+                                throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+                            }
+                        }
+                    }
+                } else if (ctx.ADD() == null) {
+                    throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+                }
+
+                caster.checkWriteCast(unaryemd);
+                utility.checkWriteBranch(ctx);
+            }
+        }
+    }
+
+    void processCast(final CastContext ctx) {
+        final ExpressionMetadata castemd = metadata.getExpressionMetadata(ctx);
+        final Object postConst = castemd.postConst;
+
+        if (postConst == null) {
+            writer.visit(ctx.expression());
+            caster.checkWriteCast(castemd);
+        } else {
+            utility.writeConstant(ctx, postConst);
+        }
+
+        utility.checkWriteBranch(ctx);
+    }
+
+    void processBinary(final BinaryContext ctx) {
+        final ExpressionMetadata binaryemd = metadata.getExpressionMetadata(ctx);
+        final Object postConst = binaryemd.postConst;
+        final Object preConst = binaryemd.preConst;
+        final Branch branch = utility.getBranch(ctx);
+
+        if (postConst != null) {
+            utility.writeConstant(ctx, postConst);
+        } else if (preConst != null) {
+            if (branch == null) {
+                utility.writeConstant(ctx, preConst);
+                caster.checkWriteCast(binaryemd);
+            } else {
+                throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+            }
+        } else if (binaryemd.from.sort == Sort.STRING) {
+            final boolean marked = utility.containsStrings(ctx);
+
+            if (!marked) {
+                utility.writeNewStrings();
+            }
+
+            final ExpressionContext exprctx0 = ctx.expression(0);
+            final ExpressionMetadata expremd0 = metadata.getExpressionMetadata(exprctx0);
+            utility.addStrings(exprctx0);
+            writer.visit(exprctx0);
+
+            if (utility.containsStrings(exprctx0)) {
+                utility.writeAppendStrings(expremd0.from.sort);
+                utility.removeStrings(exprctx0);
+            }
+
+            final ExpressionContext exprctx1 = ctx.expression(1);
+            final ExpressionMetadata expremd1 = metadata.getExpressionMetadata(exprctx1);
+            utility.addStrings(exprctx1);
+            writer.visit(exprctx1);
+
+            if (utility.containsStrings(exprctx1)) {
+                utility.writeAppendStrings(expremd1.from.sort);
+                utility.removeStrings(exprctx1);
+            }
+
+            if (marked) {
+                utility.removeStrings(ctx);
+            } else {
+                utility.writeToStrings();
+            }
+
+            caster.checkWriteCast(binaryemd);
+        } else {
+            final ExpressionContext exprctx0 = ctx.expression(0);
+            final ExpressionContext exprctx1 = ctx.expression(1);
+
+            writer.visit(exprctx0);
+            writer.visit(exprctx1);
+
+            final Type type = binaryemd.from;
+
+            if      (ctx.MUL()   != null) utility.writeBinaryInstruction(ctx, type, MUL);
+            else if (ctx.DIV()   != null) utility.writeBinaryInstruction(ctx, type, DIV);
+            else if (ctx.REM()   != null) utility.writeBinaryInstruction(ctx, type, REM);
+            else if (ctx.ADD()   != null) utility.writeBinaryInstruction(ctx, type, ADD);
+            else if (ctx.SUB()   != null) utility.writeBinaryInstruction(ctx, type, SUB);
+            else if (ctx.LSH()   != null) utility.writeBinaryInstruction(ctx, type, LSH);
+            else if (ctx.USH()   != null) utility.writeBinaryInstruction(ctx, type, USH);
+            else if (ctx.RSH()   != null) utility.writeBinaryInstruction(ctx, type, RSH);
+            else if (ctx.BWAND() != null) utility.writeBinaryInstruction(ctx, type, BWAND);
+            else if (ctx.BWXOR() != null) utility.writeBinaryInstruction(ctx, type, BWXOR);
+            else if (ctx.BWOR()  != null) utility.writeBinaryInstruction(ctx, type, BWOR);
+            else {
+                throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+            }
+
+            caster.checkWriteCast(binaryemd);
+        }
+
+        utility.checkWriteBranch(ctx);
+    }
+
+    void processComp(final CompContext ctx) {
+        final ExpressionMetadata compemd = metadata.getExpressionMetadata(ctx);
+        final Object postConst = compemd.postConst;
+        final Object preConst = compemd.preConst;
+        final Branch branch = utility.getBranch(ctx);
+
+        if (postConst != null) {
+            if (branch == null) {
+                utility.writeConstant(ctx, postConst);
+            } else {
+                if ((boolean)postConst && branch.tru != null) {
+                    execute.mark(branch.tru);
+                } else if (!(boolean)postConst && branch.fals != null) {
+                    execute.mark(branch.fals);
+                }
+            }
+        } else if (preConst != null) {
+            if (branch == null) {
+                utility.writeConstant(ctx, preConst);
+                caster.checkWriteCast(compemd);
+            } else {
+                throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+            }
+        } else {
+            final ExpressionContext exprctx0 = ctx.expression(0);
+            final ExpressionMetadata expremd0 = metadata.getExpressionMetadata(exprctx0);
+
+            final ExpressionContext exprctx1 = ctx.expression(1);
+            final ExpressionMetadata expremd1 = metadata.getExpressionMetadata(exprctx1);
+            final org.objectweb.asm.Type type = expremd1.to.type;
+            final Sort sort1 = expremd1.to.sort;
+
+            writer.visit(exprctx0);
+
+            if (!expremd1.isNull) {
+                writer.visit(exprctx1);
+            }
+
+            final boolean tru = branch != null && branch.tru != null;
+            final boolean fals = branch != null && branch.fals != null;
+            final Label jump = tru ? branch.tru : fals ? branch.fals : new Label();
+            final Label end = new Label();
+
+            final boolean eq = (ctx.EQ() != null || ctx.EQR() != null) && (tru || !fals) ||
+                (ctx.NE() != null || ctx.NER() != null) && fals;
+            final boolean ne = (ctx.NE() != null || ctx.NER() != null) && (tru || !fals) ||
+                (ctx.EQ() != null || ctx.EQR() != null) && fals;
+            final boolean lt  = ctx.LT()  != null && (tru || !fals) || ctx.GTE() != null && fals;
+            final boolean lte = ctx.LTE() != null && (tru || !fals) || ctx.GT()  != null && fals;
+            final boolean gt  = ctx.GT()  != null && (tru || !fals) || ctx.LTE() != null && fals;
+            final boolean gte = ctx.GTE() != null && (tru || !fals) || ctx.LT()  != null && fals;
+
+            boolean writejump = true;
+
+            switch (sort1) {
+                case VOID:
+                case BYTE:
+                case SHORT:
+                case CHAR:
+                    throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+                case BOOL:
+                    if      (eq) execute.ifZCmp(GeneratorAdapter.EQ, jump);
+                    else if (ne) execute.ifZCmp(GeneratorAdapter.NE, jump);
+                    else {
+                        throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+                    }
+
+                    break;
+                case INT:
+                case LONG:
+                case FLOAT:
+                case DOUBLE:
+                    if      (eq)  execute.ifCmp(type, GeneratorAdapter.EQ, jump);
+                    else if (ne)  execute.ifCmp(type, GeneratorAdapter.NE, jump);
+                    else if (lt)  execute.ifCmp(type, GeneratorAdapter.LT, jump);
+                    else if (lte) execute.ifCmp(type, GeneratorAdapter.LE, jump);
+                    else if (gt)  execute.ifCmp(type, GeneratorAdapter.GT, jump);
+                    else if (gte) execute.ifCmp(type, GeneratorAdapter.GE, jump);
+                    else {
+                        throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+                    }
+
+                    break;
+                case DEF:
+                    if (eq) {
+                        if (expremd1.isNull) {
+                            execute.ifNull(jump);
+                        } else if (!expremd0.isNull && ctx.EQ() != null) {
+                            execute.invokeStatic(definition.defobjType.type, DEF_EQ_CALL);
+                        } else {
+                            execute.ifCmp(type, GeneratorAdapter.EQ, jump);
+                        }
+                    } else if (ne) {
+                        if (expremd1.isNull) {
+                            execute.ifNonNull(jump);
+                        } else if (!expremd0.isNull && ctx.NE() != null) {
+                            execute.invokeStatic(definition.defobjType.type, DEF_EQ_CALL);
+                            execute.ifZCmp(GeneratorAdapter.EQ, jump);
+                        } else {
+                            execute.ifCmp(type, GeneratorAdapter.NE, jump);
+                        }
+                    } else if (lt) {
+                        execute.invokeStatic(definition.defobjType.type, DEF_LT_CALL);
+                    } else if (lte) {
+                        execute.invokeStatic(definition.defobjType.type, DEF_LTE_CALL);
+                    } else if (gt) {
+                        execute.invokeStatic(definition.defobjType.type, DEF_GT_CALL);
+                    } else if (gte) {
+                        execute.invokeStatic(definition.defobjType.type, DEF_GTE_CALL);
+                    } else {
+                        throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+                    }
+
+                    writejump = expremd1.isNull || ne || ctx.EQR() != null;
+
+                    if (branch != null && !writejump) {
+                        execute.ifZCmp(GeneratorAdapter.NE, jump);
+                    }
+
+                    break;
+                default:
+                    if (eq) {
+                        if (expremd1.isNull) {
+                            execute.ifNull(jump);
+                        } else if (ctx.EQ() != null) {
+                            execute.invokeStatic(definition.utilityType.type, CHECKEQUALS);
+
+                            if (branch != null) {
+                                execute.ifZCmp(GeneratorAdapter.NE, jump);
+                            }
+
+                            writejump = false;
+                        } else {
+                            execute.ifCmp(type, GeneratorAdapter.EQ, jump);
+                        }
+                    } else if (ne) {
+                        if (expremd1.isNull) {
+                            execute.ifNonNull(jump);
+                        } else if (ctx.NE() != null) {
+                            execute.invokeStatic(definition.utilityType.type, CHECKEQUALS);
+                            execute.ifZCmp(GeneratorAdapter.EQ, jump);
+                        } else {
+                            execute.ifCmp(type, GeneratorAdapter.NE, jump);
+                        }
+                    } else {
+                        throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+                    }
+            }
+
+            if (branch == null) {
+                if (writejump) {
+                    execute.push(false);
+                    execute.goTo(end);
+                    execute.mark(jump);
+                    execute.push(true);
+                    execute.mark(end);
+                }
+
+                caster.checkWriteCast(compemd);
+            }
+        }
+    }
+
+    void processBool(final BoolContext ctx) {
+        final ExpressionMetadata boolemd = metadata.getExpressionMetadata(ctx);
+        final Object postConst = boolemd.postConst;
+        final Object preConst = boolemd.preConst;
+        final Branch branch = utility.getBranch(ctx);
+
+        if (postConst != null) {
+            if (branch == null) {
+                utility.writeConstant(ctx, postConst);
+            } else {
+                if ((boolean)postConst && branch.tru != null) {
+                    execute.mark(branch.tru);
+                } else if (!(boolean)postConst && branch.fals != null) {
+                    execute.mark(branch.fals);
+                }
+            }
+        } else if (preConst != null) {
+            if (branch == null) {
+                utility.writeConstant(ctx, preConst);
+                caster.checkWriteCast(boolemd);
+            } else {
+                throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+            }
+        } else {
+            final ExpressionContext exprctx0 = ctx.expression(0);
+            final ExpressionContext exprctx1 = ctx.expression(1);
+
+            if (branch == null) {
+                if (ctx.BOOLAND() != null) {
+                    final Branch local = utility.markBranch(ctx, exprctx0, exprctx1);
+                    local.fals = new Label();
+                    final Label end = new Label();
+
+                    writer.visit(exprctx0);
+                    writer.visit(exprctx1);
+
+                    execute.push(true);
+                    execute.goTo(end);
+                    execute.mark(local.fals);
+                    execute.push(false);
+                    execute.mark(end);
+                } else if (ctx.BOOLOR() != null) {
+                    final Branch branch0 = utility.markBranch(ctx, exprctx0);
+                    branch0.tru = new Label();
+                    final Branch branch1 = utility.markBranch(ctx, exprctx1);
+                    branch1.fals = new Label();
+                    final Label aend = new Label();
+
+                    writer.visit(exprctx0);
+                    writer.visit(exprctx1);
+
+                    execute.mark(branch0.tru);
+                    execute.push(true);
+                    execute.goTo(aend);
+                    execute.mark(branch1.fals);
+                    execute.push(false);
+                    execute.mark(aend);
+                } else {
+                    throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+                }
+
+                caster.checkWriteCast(boolemd);
+            } else {
+                if (ctx.BOOLAND() != null) {
+                    final Branch branch0 = utility.markBranch(ctx, exprctx0);
+                    branch0.fals = branch.fals == null ? new Label() : branch.fals;
+                    final Branch branch1 = utility.markBranch(ctx, exprctx1);
+                    branch1.tru = branch.tru;
+                    branch1.fals = branch.fals;
+
+                    writer.visit(exprctx0);
+                    writer.visit(exprctx1);
+
+                    if (branch.fals == null) {
+                        execute.mark(branch0.fals);
+                    }
+                } else if (ctx.BOOLOR() != null) {
+                    final Branch branch0 = utility.markBranch(ctx, exprctx0);
+                    branch0.tru = branch.tru == null ? new Label() : branch.tru;
+                    final Branch branch1 = utility.markBranch(ctx, exprctx1);
+                    branch1.tru = branch.tru;
+                    branch1.fals = branch.fals;
+
+                    writer.visit(exprctx0);
+                    writer.visit(exprctx1);
+
+                    if (branch.tru == null) {
+                        execute.mark(branch0.tru);
+                    }
+                } else {
+                    throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+                }
+            }
+        }
+    }
+
+    void processConditional(final ConditionalContext ctx) {
+        final ExpressionMetadata condemd = metadata.getExpressionMetadata(ctx);
+        final Branch branch = utility.getBranch(ctx);
+
+        final ExpressionContext expr0 = ctx.expression(0);
+        final ExpressionContext expr1 = ctx.expression(1);
+        final ExpressionContext expr2 = ctx.expression(2);
+
+        final Branch local = utility.markBranch(ctx, expr0);
+        local.fals = new Label();
+        local.end = new Label();
+
+        if (branch != null) {
+            utility.copyBranch(branch, expr1, expr2);
+        }
+
+        writer.visit(expr0);
+        writer.visit(expr1);
+        execute.goTo(local.end);
+        execute.mark(local.fals);
+        writer.visit(expr2);
+        execute.mark(local.end);
+
+        if (branch == null) {
+            caster.checkWriteCast(condemd);
+        }
+    }
+
+    void processAssignment(final AssignmentContext ctx) {
+        final ExpressionMetadata expremd = metadata.getExpressionMetadata(ctx);
+        writer.visit(ctx.extstart());
+        caster.checkWriteCast(expremd);
+        utility.checkWriteBranch(ctx);
+    }
+
+    void processIncrement(final IncrementContext ctx) {
+        final ExpressionMetadata incremd = metadata.getExpressionMetadata(ctx);
+        final Object postConst = incremd.postConst;
+
+        if (postConst == null) {
+            utility.writeNumeric(ctx, incremd.preConst);
+            caster.checkWriteCast(incremd);
+        } else {
+            utility.writeConstant(ctx, postConst);
+        }
+
+        utility.checkWriteBranch(ctx);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
new file mode 100644
index 00000000000..8ab729f98fa
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
@@ -0,0 +1,769 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.elasticsearch.painless.Definition.Constructor;
+import org.elasticsearch.painless.Definition.Field;
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Metadata.ExpressionMetadata;
+import org.elasticsearch.painless.Metadata.ExtNodeMetadata;
+import org.elasticsearch.painless.Metadata.ExternalMetadata;
+import org.elasticsearch.painless.PainlessParser.ExpressionContext;
+import org.elasticsearch.painless.PainlessParser.ExtbraceContext;
+import org.elasticsearch.painless.PainlessParser.ExtcallContext;
+import org.elasticsearch.painless.PainlessParser.ExtcastContext;
+import org.elasticsearch.painless.PainlessParser.ExtdotContext;
+import org.elasticsearch.painless.PainlessParser.ExtfieldContext;
+import org.elasticsearch.painless.PainlessParser.ExtnewContext;
+import org.elasticsearch.painless.PainlessParser.ExtprecContext;
+import org.elasticsearch.painless.PainlessParser.ExtstartContext;
+import org.elasticsearch.painless.PainlessParser.ExtstringContext;
+import org.elasticsearch.painless.PainlessParser.ExttypeContext;
+import org.elasticsearch.painless.PainlessParser.ExtvarContext;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import java.util.List;
+
+import static org.elasticsearch.painless.PainlessParser.ADD;
+import static org.elasticsearch.painless.PainlessParser.DIV;
+import static org.elasticsearch.painless.PainlessParser.MUL;
+import static org.elasticsearch.painless.PainlessParser.REM;
+import static org.elasticsearch.painless.PainlessParser.SUB;
+import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
+import static org.elasticsearch.painless.WriterConstants.DEFINITION_TYPE;
+import static org.elasticsearch.painless.WriterConstants.DEF_ARRAY_LOAD;
+import static org.elasticsearch.painless.WriterConstants.DEF_ARRAY_STORE;
+import static org.elasticsearch.painless.WriterConstants.DEF_FIELD_LOAD;
+import static org.elasticsearch.painless.WriterConstants.DEF_FIELD_STORE;
+import static org.elasticsearch.painless.WriterConstants.DEF_METHOD_CALL;
+import static org.elasticsearch.painless.WriterConstants.TOBYTEEXACT_INT;
+import static org.elasticsearch.painless.WriterConstants.TOBYTEEXACT_LONG;
+import static org.elasticsearch.painless.WriterConstants.TOBYTEWOOVERFLOW_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.TOBYTEWOOVERFLOW_FLOAT;
+import static org.elasticsearch.painless.WriterConstants.TOCHAREXACT_INT;
+import static org.elasticsearch.painless.WriterConstants.TOCHAREXACT_LONG;
+import static org.elasticsearch.painless.WriterConstants.TOCHARWOOVERFLOW_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.TOCHARWOOVERFLOW_FLOAT;
+import static org.elasticsearch.painless.WriterConstants.TOFLOATWOOVERFLOW_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.TOINTEXACT_LONG;
+import static org.elasticsearch.painless.WriterConstants.TOINTWOOVERFLOW_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.TOINTWOOVERFLOW_FLOAT;
+import static org.elasticsearch.painless.WriterConstants.TOLONGWOOVERFLOW_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.TOLONGWOOVERFLOW_FLOAT;
+import static org.elasticsearch.painless.WriterConstants.TOSHORTEXACT_INT;
+import static org.elasticsearch.painless.WriterConstants.TOSHORTEXACT_LONG;
+import static org.elasticsearch.painless.WriterConstants.TOSHORTWOOVERFLOW_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.TOSHORTWOOVERFLOW_FLOAT;
+
+class WriterExternal {
+    private final Metadata metadata;
+    private final Definition definition;
+    private final CompilerSettings settings;
+
+    private final GeneratorAdapter execute;
+
+    private final Writer writer;
+    private final WriterUtility utility;
+    private final WriterCaster caster;
+
+    WriterExternal(final Metadata metadata, final GeneratorAdapter execute, final Writer writer,
+                   final WriterUtility utility, final WriterCaster caster) {
+        this.metadata = metadata;
+        definition = metadata.definition;
+        settings = metadata.settings;
+
+        this.execute = execute;
+
+        this.writer = writer;
+        this.utility = utility;
+        this.caster = caster;
+    }
+
+    void processExtstart(final ExtstartContext ctx) {
+        final ExternalMetadata startemd = metadata.getExternalMetadata(ctx);
+
+        if (startemd.token == ADD) {
+            final ExpressionMetadata storeemd = metadata.getExpressionMetadata(startemd.storeExpr);
+
+            if (startemd.current.sort == Sort.STRING || storeemd.from.sort == Sort.STRING) {
+                utility.writeNewStrings();
+                utility.addStrings(startemd.storeExpr);
+            }
+        }
+
+        final ExtprecContext precctx = ctx.extprec();
+        final ExtcastContext castctx = ctx.extcast();
+        final ExttypeContext typectx = ctx.exttype();
+        final ExtvarContext varctx = ctx.extvar();
+        final ExtnewContext newctx = ctx.extnew();
+        final ExtstringContext stringctx = ctx.extstring();
+
+        if (precctx != null) {
+            writer.visit(precctx);
+        } else if (castctx != null) {
+            writer.visit(castctx);
+        } else if (typectx != null) {
+            writer.visit(typectx);
+        } else if (varctx != null) {
+            writer.visit(varctx);
+        } else if (newctx != null) {
+            writer.visit(newctx);
+        } else if (stringctx != null) {
+            writer.visit(stringctx);
+        } else {
+            throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+        }
+    }
+
+    void processExtprec(final ExtprecContext ctx) {
+        final ExtprecContext precctx = ctx.extprec();
+        final ExtcastContext castctx = ctx.extcast();
+        final ExttypeContext typectx = ctx.exttype();
+        final ExtvarContext varctx = ctx.extvar();
+        final ExtnewContext newctx = ctx.extnew();
+        final ExtstringContext stringctx = ctx.extstring();
+
+        if (precctx != null) {
+            writer.visit(precctx);
+        } else if (castctx != null) {
+            writer.visit(castctx);
+        } else if (typectx != null) {
+            writer.visit(typectx);
+        } else if (varctx != null) {
+            writer.visit(varctx);
+        } else if (newctx != null) {
+            writer.visit(newctx);
+        } else if (stringctx != null) {
+            writer.visit(stringctx);
+        } else {
+            throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+        }
+
+        final ExtdotContext dotctx = ctx.extdot();
+        final ExtbraceContext bracectx = ctx.extbrace();
+
+        if (dotctx != null) {
+            writer.visit(dotctx);
+        } else if (bracectx != null) {
+            writer.visit(bracectx);
+        }
+    }
+
+    void processExtcast(final ExtcastContext ctx) {
+        ExtNodeMetadata castenmd = metadata.getExtNodeMetadata(ctx);
+
+        final ExtprecContext precctx = ctx.extprec();
+        final ExtcastContext castctx = ctx.extcast();
+        final ExttypeContext typectx = ctx.exttype();
+        final ExtvarContext varctx = ctx.extvar();
+        final ExtnewContext newctx = ctx.extnew();
+        final ExtstringContext stringctx = ctx.extstring();
+
+        if (precctx != null) {
+            writer.visit(precctx);
+        } else if (castctx != null) {
+            writer.visit(castctx);
+        } else if (typectx != null) {
+            writer.visit(typectx);
+        } else if (varctx != null) {
+            writer.visit(varctx);
+        } else if (newctx != null) {
+            writer.visit(newctx);
+        } else if (stringctx != null) {
+            writer.visit(stringctx);
+        } else {
+            throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+        }
+
+        caster.checkWriteCast(ctx, castenmd.castTo);
+    }
+
+    void processExtbrace(final ExtbraceContext ctx) {
+        final ExpressionContext exprctx = ctx.expression();
+
+        writer.visit(exprctx);
+        writeLoadStoreExternal(ctx);
+
+        final ExtdotContext dotctx = ctx.extdot();
+        final ExtbraceContext bracectx = ctx.extbrace();
+
+        if (dotctx != null) {
+            writer.visit(dotctx);
+        } else if (bracectx != null) {
+            writer.visit(bracectx);
+        }
+    }
+
+    void processExtdot(final ExtdotContext ctx) {
+        final ExtcallContext callctx = ctx.extcall();
+        final ExtfieldContext fieldctx = ctx.extfield();
+
+        if (callctx != null) {
+            writer.visit(callctx);
+        } else if (fieldctx != null) {
+            writer.visit(fieldctx);
+        }
+    }
+
+    void processExttype(final ExttypeContext ctx) {
+        writer.visit(ctx.extdot());
+    }
+
+    void processExtcall(final ExtcallContext ctx) {
+        writeCallExternal(ctx);
+
+        final ExtdotContext dotctx = ctx.extdot();
+        final ExtbraceContext bracectx = ctx.extbrace();
+
+        if (dotctx != null) {
+            writer.visit(dotctx);
+        } else if (bracectx != null) {
+            writer.visit(bracectx);
+        }
+    }
+
+    void processExtvar(final ExtvarContext ctx) {
+        writeLoadStoreExternal(ctx);
+
+        final ExtdotContext dotctx = ctx.extdot();
+        final ExtbraceContext bracectx = ctx.extbrace();
+
+        if (dotctx != null) {
+            writer.visit(dotctx);
+        } else if (bracectx != null) {
+            writer.visit(bracectx);
+        }
+    }
+
+    void processExtfield(final ExtfieldContext ctx) {
+        writeLoadStoreExternal(ctx);
+
+        final ExtdotContext dotctx = ctx.extdot();
+        final ExtbraceContext bracectx = ctx.extbrace();
+
+        if (dotctx != null) {
+            writer.visit(dotctx);
+        } else if (bracectx != null) {
+            writer.visit(bracectx);
+        }
+    }
+
+    void processExtnew(final ExtnewContext ctx) {
+        writeNewExternal(ctx);
+
+        final ExtdotContext dotctx = ctx.extdot();
+        final ExtbraceContext bracectx = ctx.extbrace();
+
+        if (dotctx != null) {
+            writer.visit(dotctx);
+        } else if (bracectx != null) {
+            writer.visit(bracectx);
+        }
+    }
+
+    void processExtstring(final ExtstringContext ctx) {
+        final ExtNodeMetadata stringenmd = metadata.getExtNodeMetadata(ctx);
+
+        utility.writeConstant(ctx, stringenmd.target);
+
+        final ExtdotContext dotctx = ctx.extdot();
+        final ExtbraceContext bracectx = ctx.extbrace();
+
+        if (dotctx != null) {
+            writer.visit(dotctx);
+        } else if (bracectx != null) {
+            writer.visit(bracectx);
+        }
+    }
+
+    private void writeLoadStoreExternal(final ParserRuleContext source) {
+        final ExtNodeMetadata sourceenmd = metadata.getExtNodeMetadata(source);
+        final ExternalMetadata parentemd = metadata.getExternalMetadata(sourceenmd.parent);
+
+        final boolean length = "#length".equals(sourceenmd.target);
+        final boolean array = "#brace".equals(sourceenmd.target);
+        final boolean name = sourceenmd.target instanceof String && !length && !array;
+        final boolean variable = sourceenmd.target instanceof Integer;
+        final boolean field = sourceenmd.target instanceof Field;
+        final boolean shortcut = sourceenmd.target instanceof Object[];
+
+        if (!length && !variable && !field && !array && !name && !shortcut) {
+            throw new IllegalStateException(WriterUtility.error(source) + "Target not found for load/store.");
+        }
+
+        final boolean maplist = shortcut && (boolean)((Object[])sourceenmd.target)[2];
+        final Object constant = shortcut ? ((Object[])sourceenmd.target)[3] : null;
+
+        final boolean x1 = field || name || (shortcut && !maplist);
+        final boolean x2 = array || (shortcut && maplist);
+
+        if (length) {
+            execute.arrayLength();
+        } else if (sourceenmd.last && parentemd.storeExpr != null) {
+            final ExpressionMetadata expremd = metadata.getExpressionMetadata(parentemd.storeExpr);
+            final boolean cat = utility.containsStrings(parentemd.storeExpr);
+
+            if (cat) {
+                if (field || name || shortcut) {
+                    execute.dupX1();
+                } else if (array) {
+                    execute.dup2X1();
+                }
+
+                if (maplist) {
+                    if (constant != null) {
+                        utility.writeConstant(source, constant);
+                    }
+
+                    execute.dupX2();
+                }
+
+                writeLoadStoreInstruction(source, false, variable, field, name, array, shortcut);
+                utility.writeAppendStrings(sourceenmd.type.sort);
+                writer.visit(parentemd.storeExpr);
+
+                if (utility.containsStrings(parentemd.storeExpr)) {
+                    utility.writeAppendStrings(expremd.to.sort);
+                    utility.removeStrings(parentemd.storeExpr);
+                }
+
+                utility.writeToStrings();
+                caster.checkWriteCast(source, sourceenmd.castTo);
+
+                if (parentemd.read) {
+                    utility.writeDup(sourceenmd.type.sort.size, x1, x2);
+                }
+
+                writeLoadStoreInstruction(source, true, variable, field, name, array, shortcut);
+            } else if (parentemd.token > 0) {
+                final int token = parentemd.token;
+
+                if (field || name || shortcut) {
+                    execute.dup();
+                } else if (array) {
+                    execute.dup2();
+                }
+
+                if (maplist) {
+                    if (constant != null) {
+                        utility.writeConstant(source, constant);
+                    }
+
+                    execute.dupX1();
+                }
+
+                writeLoadStoreInstruction(source, false, variable, field, name, array, shortcut);
+
+                if (parentemd.read && parentemd.post) {
+                    utility.writeDup(sourceenmd.type.sort.size, x1, x2);
+                }
+
+                caster.checkWriteCast(source, sourceenmd.castFrom);
+                writer.visit(parentemd.storeExpr);
+
+                utility.writeBinaryInstruction(source, sourceenmd.promote, token);
+
+                boolean exact = false;
+
+                if (!settings.getNumericOverflow() && expremd.typesafe && sourceenmd.type.sort != Sort.DEF &&
+                    (token == MUL || token == DIV || token == REM || token == ADD || token == SUB)) {
+                    exact = writeExactInstruction(sourceenmd.type.sort, sourceenmd.promote.sort);
+                }
+
+                if (!exact) {
+                    caster.checkWriteCast(source, sourceenmd.castTo);
+                }
+
+                if (parentemd.read && !parentemd.post) {
+                    utility.writeDup(sourceenmd.type.sort.size, x1, x2);
+                }
+
+                writeLoadStoreInstruction(source, true, variable, field, name, array, shortcut);
+            } else {
+                if (constant != null) {
+                    utility.writeConstant(source, constant);
+                }
+
+                writer.visit(parentemd.storeExpr);
+
+                if (parentemd.read) {
+                    utility.writeDup(sourceenmd.type.sort.size, x1, x2);
+                }
+
+                writeLoadStoreInstruction(source, true, variable, field, name, array, shortcut);
+            }
+        } else {
+            if (constant != null) {
+                utility.writeConstant(source, constant);
+            }
+
+            writeLoadStoreInstruction(source, false, variable, field, name, array, shortcut);
+        }
+    }
+
+    private void writeLoadStoreInstruction(final ParserRuleContext source,
+                                           final boolean store, final boolean variable,
+                                           final boolean field, final boolean name,
+                                           final boolean array, final boolean shortcut) {
+        final ExtNodeMetadata sourceemd = metadata.getExtNodeMetadata(source);
+
+        if (variable) {
+            writeLoadStoreVariable(source, store, sourceemd.type, (int)sourceemd.target);
+        } else if (field) {
+            writeLoadStoreField(store, (Field)sourceemd.target);
+        } else if (name) {
+            writeLoadStoreField(source, store, (String)sourceemd.target);
+        } else if (array) {
+            writeLoadStoreArray(source, store, sourceemd.type);
+        } else if (shortcut) {
+            Object[] targets = (Object[])sourceemd.target;
+            writeLoadStoreShortcut(store, (Method)targets[0], (Method)targets[1]);
+        } else {
+            throw new IllegalStateException(WriterUtility.error(source) + "Load/Store requires a variable, field, or array.");
+        }
+    }
+
+    private void writeLoadStoreVariable(final ParserRuleContext source, final boolean store,
+                                        final Type type, final int slot) {
+        if (type.sort == Sort.VOID) {
+            throw new IllegalStateException(WriterUtility.error(source) + "Cannot load/store void type.");
+        }
+
+        if (store) {
+            execute.visitVarInsn(type.type.getOpcode(Opcodes.ISTORE), slot);
+        } else {
+            execute.visitVarInsn(type.type.getOpcode(Opcodes.ILOAD), slot);
+        }
+    }
+
+    private void writeLoadStoreField(final boolean store, final Field field) {
+        if (java.lang.reflect.Modifier.isStatic(field.reflect.getModifiers())) {
+            if (store) {
+                execute.putStatic(field.owner.type, field.reflect.getName(), field.type.type);
+            } else {
+                execute.getStatic(field.owner.type, field.reflect.getName(), field.type.type);
+
+                if (!field.generic.clazz.equals(field.type.clazz)) {
+                    execute.checkCast(field.generic.type);
+                }
+            }
+        } else {
+            if (store) {
+                execute.putField(field.owner.type, field.reflect.getName(), field.type.type);
+            } else {
+                execute.getField(field.owner.type, field.reflect.getName(), field.type.type);
+
+                if (!field.generic.clazz.equals(field.type.clazz)) {
+                    execute.checkCast(field.generic.type);
+                }
+            }
+        }
+    }
+
+    private void writeLoadStoreField(final ParserRuleContext source, final boolean store, final String name) {
+        if (store) {
+            final ExtNodeMetadata sourceemd = metadata.getExtNodeMetadata(source);
+            final ExternalMetadata parentemd = metadata.getExternalMetadata(sourceemd.parent);
+            final ExpressionMetadata expremd = metadata.getExpressionMetadata(parentemd.storeExpr);
+
+            execute.push(name);
+            execute.loadThis();
+            execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE);
+            execute.push(parentemd.token == 0 && expremd.typesafe);
+            execute.invokeStatic(definition.defobjType.type, DEF_FIELD_STORE);
+        } else {
+            execute.push(name);
+            execute.loadThis();
+            execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE);
+            execute.invokeStatic(definition.defobjType.type, DEF_FIELD_LOAD);
+        }
+    }
+
+    private void writeLoadStoreArray(final ParserRuleContext source, final boolean store, final Type type) {
+        if (type.sort == Sort.VOID) {
+            throw new IllegalStateException(WriterUtility.error(source) + "Cannot load/store void type.");
+        }
+
+        if (type.sort == Sort.DEF) {
+            final ExtbraceContext bracectx = (ExtbraceContext)source;
+            final ExpressionMetadata expremd0 = metadata.getExpressionMetadata(bracectx.expression());
+
+            if (store) {
+                final ExtNodeMetadata braceenmd = metadata.getExtNodeMetadata(bracectx);
+                final ExternalMetadata parentemd = metadata.getExternalMetadata(braceenmd.parent);
+                final ExpressionMetadata expremd1 = metadata.getExpressionMetadata(parentemd.storeExpr);
+
+                execute.loadThis();
+                execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE);
+                execute.push(expremd0.typesafe);
+                execute.push(parentemd.token == 0 && expremd1.typesafe);
+                execute.invokeStatic(definition.defobjType.type, DEF_ARRAY_STORE);
+            } else {
+                execute.loadThis();
+                execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE);
+                execute.push(expremd0.typesafe);
+                execute.invokeStatic(definition.defobjType.type, DEF_ARRAY_LOAD);
+            }
+        } else {
+            if (store) {
+                execute.arrayStore(type.type);
+            } else {
+                execute.arrayLoad(type.type);
+            }
+        }
+    }
+
+    private void writeLoadStoreShortcut(final boolean store, final Method getter, final Method setter) {
+        final Method method = store ? setter : getter;
+
+        if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
+            execute.invokeInterface(method.owner.type, method.method);
+        } else {
+            execute.invokeVirtual(method.owner.type, method.method);
+        }
+
+        if (store) {
+            utility.writePop(method.rtn.type.getSize());
+        } else if (!method.rtn.clazz.equals(method.handle.type().returnType())) {
+            execute.checkCast(method.rtn.type);
+        }
+    }
+
+    /**
+     * Called for any compound assignment (including increment/decrement instructions).
+     * We have to be stricter than writeBinary, and do overflow checks against the original type's size
+     * instead of the promoted type's size, since the result will be implicitly cast back.
+     *
+     * @return This will be true if an instruction is written, false otherwise.
+     */
+    private boolean writeExactInstruction(final Sort osort, final Sort psort) {
+        if (psort == Sort.DOUBLE) {
+            if (osort == Sort.FLOAT) {
+                execute.invokeStatic(definition.utilityType.type, TOFLOATWOOVERFLOW_DOUBLE);
+            } else if (osort == Sort.FLOAT_OBJ) {
+                execute.invokeStatic(definition.utilityType.type, TOFLOATWOOVERFLOW_DOUBLE);
+                execute.checkCast(definition.floatobjType.type);
+            } else if (osort == Sort.LONG) {
+                execute.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_DOUBLE);
+            } else if (osort == Sort.LONG_OBJ) {
+                execute.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_DOUBLE);
+                execute.checkCast(definition.longobjType.type);
+            } else if (osort == Sort.INT) {
+                execute.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_DOUBLE);
+            } else if (osort == Sort.INT_OBJ) {
+                execute.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_DOUBLE);
+                execute.checkCast(definition.intobjType.type);
+            } else if (osort == Sort.CHAR) {
+                execute.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_DOUBLE);
+            } else if (osort == Sort.CHAR_OBJ) {
+                execute.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_DOUBLE);
+                execute.checkCast(definition.charobjType.type);
+            } else if (osort == Sort.SHORT) {
+                execute.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_DOUBLE);
+            } else if (osort == Sort.SHORT_OBJ) {
+                execute.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_DOUBLE);
+                execute.checkCast(definition.shortobjType.type);
+            } else if (osort == Sort.BYTE) {
+                execute.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_DOUBLE);
+            } else if (osort == Sort.BYTE_OBJ) {
+                execute.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_DOUBLE);
+                execute.checkCast(definition.byteobjType.type);
+            } else {
+                return false;
+            }
+        } else if (psort == Sort.FLOAT) {
+            if (osort == Sort.LONG) {
+                execute.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_FLOAT);
+            } else if (osort == Sort.LONG_OBJ) {
+                execute.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_FLOAT);
+                execute.checkCast(definition.longobjType.type);
+            } else if (osort == Sort.INT) {
+                execute.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_FLOAT);
+            } else if (osort == Sort.INT_OBJ) {
+                execute.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_FLOAT);
+                execute.checkCast(definition.intobjType.type);
+            } else if (osort == Sort.CHAR) {
+                execute.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_FLOAT);
+            } else if (osort == Sort.CHAR_OBJ) {
+                execute.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_FLOAT);
+                execute.checkCast(definition.charobjType.type);
+            } else if (osort == Sort.SHORT) {
+                execute.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_FLOAT);
+            } else if (osort == Sort.SHORT_OBJ) {
+                execute.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_FLOAT);
+                execute.checkCast(definition.shortobjType.type);
+            } else if (osort == Sort.BYTE) {
+                execute.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_FLOAT);
+            } else if (osort == Sort.BYTE_OBJ) {
+                execute.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_FLOAT);
+                execute.checkCast(definition.byteobjType.type);
+            } else {
+                return false;
+            }
+        } else if (psort == Sort.LONG) {
+            if (osort == Sort.INT) {
+                execute.invokeStatic(definition.mathType.type, TOINTEXACT_LONG);
+            } else if (osort == Sort.INT_OBJ) {
+                execute.invokeStatic(definition.mathType.type, TOINTEXACT_LONG);
+                execute.checkCast(definition.intobjType.type);
+            } else if (osort == Sort.CHAR) {
+                execute.invokeStatic(definition.utilityType.type, TOCHAREXACT_LONG);
+            } else if (osort == Sort.CHAR_OBJ) {
+                execute.invokeStatic(definition.utilityType.type, TOCHAREXACT_LONG);
+                execute.checkCast(definition.charobjType.type);
+            } else if (osort == Sort.SHORT) {
+                execute.invokeStatic(definition.utilityType.type, TOSHORTEXACT_LONG);
+            } else if (osort == Sort.SHORT_OBJ) {
+                execute.invokeStatic(definition.utilityType.type, TOSHORTEXACT_LONG);
+                execute.checkCast(definition.shortobjType.type);
+            } else if (osort == Sort.BYTE) {
+                execute.invokeStatic(definition.utilityType.type, TOBYTEEXACT_LONG);
+            } else if (osort == Sort.BYTE_OBJ) {
+                execute.invokeStatic(definition.utilityType.type, TOBYTEEXACT_LONG);
+                execute.checkCast(definition.byteobjType.type);
+            } else {
+                return false;
+            }
+        } else if (psort == Sort.INT) {
+            if (osort == Sort.CHAR) {
+                execute.invokeStatic(definition.utilityType.type, TOCHAREXACT_INT);
+            } else if (osort == Sort.CHAR_OBJ) {
+                execute.invokeStatic(definition.utilityType.type, TOCHAREXACT_INT);
+                execute.checkCast(definition.charobjType.type);
+            } else if (osort == Sort.SHORT) {
+                execute.invokeStatic(definition.utilityType.type, TOSHORTEXACT_INT);
+            } else if (osort == Sort.SHORT_OBJ) {
+                execute.invokeStatic(definition.utilityType.type, TOSHORTEXACT_INT);
+                execute.checkCast(definition.shortobjType.type);
+            } else if (osort == Sort.BYTE) {
+                execute.invokeStatic(definition.utilityType.type, TOBYTEEXACT_INT);
+            } else if (osort == Sort.BYTE_OBJ) {
+                execute.invokeStatic(definition.utilityType.type, TOBYTEEXACT_INT);
+                execute.checkCast(definition.byteobjType.type);
+            } else {
+                return false;
+            }
+        } else {
+            return false;
+        }
+
+        return true;
+    }
+
+    private void writeNewExternal(final ExtnewContext source) {
+        final ExtNodeMetadata sourceenmd = metadata.getExtNodeMetadata(source);
+        final ExternalMetadata parentemd = metadata.getExternalMetadata(sourceenmd.parent);
+
+        final boolean makearray = "#makearray".equals(sourceenmd.target);
+        final boolean constructor = sourceenmd.target instanceof Constructor;
+
+        if (!makearray && !constructor) {
+            throw new IllegalStateException(WriterUtility.error(source) + "Target not found for new call.");
+        }
+
+        if (makearray) {
+            for (final ExpressionContext exprctx : source.expression()) {
+                writer.visit(exprctx);
+            }
+
+            if (sourceenmd.type.sort == Sort.ARRAY) {
+                execute.visitMultiANewArrayInsn(sourceenmd.type.type.getDescriptor(), sourceenmd.type.type.getDimensions());
+            } else {
+                execute.newArray(sourceenmd.type.type);
+            }
+        } else {
+            execute.newInstance(sourceenmd.type.type);
+
+            if (parentemd.read) {
+                execute.dup();
+            }
+
+            for (final ExpressionContext exprctx : source.arguments().expression()) {
+                writer.visit(exprctx);
+            }
+
+            final Constructor target = (Constructor)sourceenmd.target;
+            execute.invokeConstructor(target.owner.type, target.method);
+        }
+    }
+
+    private void writeCallExternal(final ExtcallContext source) {
+        final ExtNodeMetadata sourceenmd = metadata.getExtNodeMetadata(source);
+
+        final boolean method = sourceenmd.target instanceof Method;
+        final boolean def = sourceenmd.target instanceof String;
+
+        if (!method && !def) {
+            throw new IllegalStateException(WriterUtility.error(source) + "Target not found for call.");
+        }
+
+        final List<ExpressionContext> arguments = source.arguments().expression();
+
+        if (method) {
+            for (final ExpressionContext exprctx : arguments) {
+                writer.visit(exprctx);
+            }
+
+            final Method target = (Method)sourceenmd.target;
+
+            if (java.lang.reflect.Modifier.isStatic(target.reflect.getModifiers())) {
+                execute.invokeStatic(target.owner.type, target.method);
+            } else if (java.lang.reflect.Modifier.isInterface(target.owner.clazz.getModifiers())) {
+                execute.invokeInterface(target.owner.type, target.method);
+            } else {
+                execute.invokeVirtual(target.owner.type, target.method);
+            }
+
+            if (!target.rtn.clazz.equals(target.handle.type().returnType())) {
+                execute.checkCast(target.rtn.type);
+            }
+        } else {
+            execute.push((String)sourceenmd.target);
+            execute.loadThis();
+            execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE);
+
+            execute.push(arguments.size());
+            execute.newArray(definition.defType.type);
+
+            for (int argument = 0; argument < arguments.size(); ++argument) {
+                execute.dup();
+                execute.push(argument);
+                writer.visit(arguments.get(argument));
+                execute.arrayStore(definition.defType.type);
+            }
+
+            execute.push(arguments.size());
+            execute.newArray(definition.booleanType.type);
+
+            for (int argument = 0; argument < arguments.size(); ++argument) {
+                execute.dup();
+                execute.push(argument);
+                execute.push(metadata.getExpressionMetadata(arguments.get(argument)).typesafe);
+                execute.arrayStore(definition.booleanType.type);
+            }
+
+            execute.invokeStatic(definition.defobjType.type, DEF_METHOD_CALL);
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterStatement.java
new file mode 100644
index 00000000000..a0e70f319b5
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterStatement.java
@@ -0,0 +1,391 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Metadata.ExpressionMetadata;
+import org.elasticsearch.painless.Metadata.StatementMetadata;
+import org.elasticsearch.painless.PainlessParser.AfterthoughtContext;
+import org.elasticsearch.painless.PainlessParser.BlockContext;
+import org.elasticsearch.painless.PainlessParser.DeclContext;
+import org.elasticsearch.painless.PainlessParser.DeclarationContext;
+import org.elasticsearch.painless.PainlessParser.DecltypeContext;
+import org.elasticsearch.painless.PainlessParser.DeclvarContext;
+import org.elasticsearch.painless.PainlessParser.DoContext;
+import org.elasticsearch.painless.PainlessParser.EmptyscopeContext;
+import org.elasticsearch.painless.PainlessParser.ExprContext;
+import org.elasticsearch.painless.PainlessParser.ExpressionContext;
+import org.elasticsearch.painless.PainlessParser.ForContext;
+import org.elasticsearch.painless.PainlessParser.IfContext;
+import org.elasticsearch.painless.PainlessParser.InitializerContext;
+import org.elasticsearch.painless.PainlessParser.MultipleContext;
+import org.elasticsearch.painless.PainlessParser.ReturnContext;
+import org.elasticsearch.painless.PainlessParser.SingleContext;
+import org.elasticsearch.painless.PainlessParser.SourceContext;
+import org.elasticsearch.painless.PainlessParser.StatementContext;
+import org.elasticsearch.painless.PainlessParser.ThrowContext;
+import org.elasticsearch.painless.PainlessParser.TrapContext;
+import org.elasticsearch.painless.PainlessParser.TryContext;
+import org.elasticsearch.painless.PainlessParser.WhileContext;
+import org.elasticsearch.painless.WriterUtility.Branch;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import static org.elasticsearch.painless.WriterConstants.PAINLESS_ERROR_TYPE;
+
+class WriterStatement {
+    private final Metadata metadata;
+
+    private final GeneratorAdapter execute;
+
+    private final Writer writer;
+    private final WriterUtility utility;
+
+    WriterStatement(final Metadata metadata, final GeneratorAdapter execute,
+                    final Writer writer, final WriterUtility utility) {
+        this.metadata = metadata;
+
+        this.execute = execute;
+
+        this.writer = writer;
+        this.utility = utility;
+    }
+
+    void processSource(final SourceContext ctx) {
+        final StatementMetadata sourcesmd = metadata.getStatementMetadata(ctx);
+
+        for (final StatementContext sctx : ctx.statement()) {
+            writer.visit(sctx);
+        }
+
+        if (!sourcesmd.methodEscape) {
+            execute.visitInsn(Opcodes.ACONST_NULL);
+            execute.returnValue();
+        }
+    }
+
+    void processIf(final IfContext ctx) {
+        final ExpressionContext exprctx = ctx.expression();
+        final boolean els = ctx.ELSE() != null;
+        final Branch branch = utility.markBranch(ctx, exprctx);
+        branch.end = new Label();
+        branch.fals = els ? new Label() : branch.end;
+
+        writer.visit(exprctx);
+
+        final BlockContext blockctx0 = ctx.block(0);
+        final StatementMetadata blockmd0 = metadata.getStatementMetadata(blockctx0);
+        writer.visit(blockctx0);
+
+        if (els) {
+            if (!blockmd0.allLast) {
+                execute.goTo(branch.end);
+            }
+
+            execute.mark(branch.fals);
+            writer.visit(ctx.block(1));
+        }
+
+        execute.mark(branch.end);
+    }
+
+    void processWhile(final WhileContext ctx) {
+        final ExpressionContext exprctx = ctx.expression();
+        final Branch branch = utility.markBranch(ctx, exprctx);
+        branch.begin = new Label();
+        branch.end = new Label();
+        branch.fals = branch.end;
+
+        utility.pushJump(branch);
+        execute.mark(branch.begin);
+        writer.visit(exprctx);
+
+        final BlockContext blockctx = ctx.block();
+        boolean allLast = false;
+
+        if (blockctx != null) {
+            final StatementMetadata blocksmd = metadata.getStatementMetadata(blockctx);
+            allLast = blocksmd.allLast;
+            writeLoopCounter(blocksmd.count > 0 ? blocksmd.count : 1);
+            writer.visit(blockctx);
+        } else if (ctx.empty() != null) {
+            writeLoopCounter(1);
+        } else {
+            throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+        }
+
+        if (!allLast) {
+            execute.goTo(branch.begin);
+        }
+
+        execute.mark(branch.end);
+        utility.popJump();
+    }
+
+    void processDo(final DoContext ctx) {
+        final ExpressionContext exprctx = ctx.expression();
+        final Branch branch = utility.markBranch(ctx, exprctx);
+        Label start = new Label();
+        branch.begin = new Label();
+        branch.end = new Label();
+        branch.fals = branch.end;
+
+        final BlockContext blockctx = ctx.block();
+        final StatementMetadata blocksmd = metadata.getStatementMetadata(blockctx);
+
+        utility.pushJump(branch);
+        execute.mark(start);
+        writer.visit(blockctx);
+        execute.mark(branch.begin);
+        writer.visit(exprctx);
+        writeLoopCounter(blocksmd.count > 0 ? blocksmd.count : 1);
+        execute.goTo(start);
+        execute.mark(branch.end);
+        utility.popJump();
+    }
+
+    void processFor(final ForContext ctx) {
+        final ExpressionContext exprctx = ctx.expression();
+        final AfterthoughtContext atctx = ctx.afterthought();
+        final Branch branch = utility.markBranch(ctx, exprctx);
+        final Label start = new Label();
+        branch.begin = atctx == null ? start : new Label();
+        branch.end = new Label();
+        branch.fals = branch.end;
+
+        utility.pushJump(branch);
+
+        if (ctx.initializer() != null) {
+            writer.visit(ctx.initializer());
+        }
+
+        execute.mark(start);
+
+        if (exprctx != null) {
+            writer.visit(exprctx);
+        }
+
+        final BlockContext blockctx = ctx.block();
+        boolean allLast = false;
+
+        if (blockctx != null) {
+            StatementMetadata blocksmd = metadata.getStatementMetadata(blockctx);
+            allLast = blocksmd.allLast;
+
+            int count = blocksmd.count > 0 ? blocksmd.count : 1;
+
+            if (atctx != null) {
+                ++count;
+            }
+
+            writeLoopCounter(count);
+            writer.visit(blockctx);
+        } else if (ctx.empty() != null) {
+            writeLoopCounter(1);
+        } else {
+            throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+        }
+
+        if (atctx != null) {
+            execute.mark(branch.begin);
+            writer.visit(atctx);
+        }
+
+        if (atctx != null || !allLast) {
+            execute.goTo(start);
+        }
+
+        execute.mark(branch.end);
+        utility.popJump();
+    }
+
+    void processDecl(final DeclContext ctx) {
+        writer.visit(ctx.declaration());
+    }
+
+    void processContinue() {
+        final Branch jump = utility.peekJump();
+        execute.goTo(jump.begin);
+    }
+
+    void processBreak() {
+        final Branch jump = utility.peekJump();
+        execute.goTo(jump.end);
+    }
+
+    void processReturn(final ReturnContext ctx) {
+        writer.visit(ctx.expression());
+        execute.returnValue();
+    }
+
+    void processTry(final TryContext ctx) {
+        final TrapContext[] trapctxs = new TrapContext[ctx.trap().size()];
+        ctx.trap().toArray(trapctxs);
+        final Branch branch = utility.markBranch(ctx, trapctxs);
+
+        Label end = new Label();
+        branch.begin = new Label();
+        branch.end = new Label();
+        branch.tru = trapctxs.length > 1 ? end : null;
+
+        execute.mark(branch.begin);
+
+        final BlockContext blockctx = ctx.block();
+        final StatementMetadata blocksmd = metadata.getStatementMetadata(blockctx);
+        writer.visit(blockctx);
+
+        if (!blocksmd.allLast) {
+            execute.goTo(end);
+        }
+
+        execute.mark(branch.end);
+
+        for (final TrapContext trapctx : trapctxs) {
+            writer.visit(trapctx);
+        }
+
+        if (!blocksmd.allLast || trapctxs.length > 1) {
+            execute.mark(end);
+        }
+    }
+
+    void processThrow(final ThrowContext ctx) {
+        writer.visit(ctx.expression());
+        execute.throwException();
+    }
+
+    void processExpr(final ExprContext ctx) {
+        final StatementMetadata exprsmd = metadata.getStatementMetadata(ctx);
+        final ExpressionContext exprctx = ctx.expression();
+        final ExpressionMetadata expremd = metadata.getExpressionMetadata(exprctx);
+        writer.visit(exprctx);
+
+        if (exprsmd.methodEscape) {
+            execute.returnValue();
+        } else {
+            utility.writePop(expremd.to.type.getSize());
+        }
+    }
+
+    void processMultiple(final MultipleContext ctx) {
+        for (final StatementContext sctx : ctx.statement()) {
+            writer.visit(sctx);
+        }
+    }
+
+    void processSingle(final SingleContext ctx) {
+        writer.visit(ctx.statement());
+    }
+
+    void processInitializer(InitializerContext ctx) {
+        final DeclarationContext declctx = ctx.declaration();
+        final ExpressionContext exprctx = ctx.expression();
+
+        if (declctx != null) {
+            writer.visit(declctx);
+        } else if (exprctx != null) {
+            final ExpressionMetadata expremd = metadata.getExpressionMetadata(exprctx);
+            writer.visit(exprctx);
+            utility.writePop(expremd.to.type.getSize());
+        } else {
+            throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+        }
+    }
+
+    void processAfterthought(AfterthoughtContext ctx) {
+        final ExpressionContext exprctx = ctx.expression();
+        final ExpressionMetadata expremd = metadata.getExpressionMetadata(exprctx);
+        writer.visit(ctx.expression());
+        utility.writePop(expremd.to.type.getSize());
+    }
+
+    void processDeclaration(DeclarationContext ctx) {
+        for (final DeclvarContext declctx : ctx.declvar()) {
+            writer.visit(declctx);
+        }
+    }
+
+    void processDeclvar(final DeclvarContext ctx) {
+        final ExpressionMetadata declvaremd = metadata.getExpressionMetadata(ctx);
+        final org.objectweb.asm.Type type = declvaremd.to.type;
+        final Sort sort = declvaremd.to.sort;
+        final int slot = (int)declvaremd.postConst;
+
+        final ExpressionContext exprctx = ctx.expression();
+        final boolean initialize = exprctx == null;
+
+        if (!initialize) {
+            writer.visit(exprctx);
+        }
+
+        switch (sort) {
+            case VOID:   throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+            case BOOL:
+            case BYTE:
+            case SHORT:
+            case CHAR:
+            case INT:    if (initialize) execute.push(0);    break;
+            case LONG:   if (initialize) execute.push(0L);   break;
+            case FLOAT:  if (initialize) execute.push(0.0F); break;
+            case DOUBLE: if (initialize) execute.push(0.0);  break;
+            default:     if (initialize) execute.visitInsn(Opcodes.ACONST_NULL);
+        }
+
+        execute.visitVarInsn(type.getOpcode(Opcodes.ISTORE), slot);
+    }
+
+    void processTrap(final TrapContext ctx) {
+        final StatementMetadata trapsmd = metadata.getStatementMetadata(ctx);
+
+        final Branch branch = utility.getBranch(ctx);
+        final Label jump = new Label();
+
+        final BlockContext blockctx = ctx.block();
+        final EmptyscopeContext emptyctx = ctx.emptyscope();
+
+        execute.mark(jump);
+        execute.visitVarInsn(trapsmd.exception.type.getOpcode(Opcodes.ISTORE), trapsmd.slot);
+
+        if (blockctx != null) {
+            writer.visit(ctx.block());
+        } else if (emptyctx == null) {
+            throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
+        }
+
+        execute.visitTryCatchBlock(branch.begin, branch.end, jump, trapsmd.exception.type.getInternalName());
+
+        if (branch.tru != null && !trapsmd.allLast) {
+            execute.goTo(branch.tru);
+        }
+    }
+
+    private void writeLoopCounter(final int count) {
+        final Label end = new Label();
+
+        execute.iinc(metadata.loopCounterSlot, -count);
+        execute.visitVarInsn(Opcodes.ILOAD, metadata.loopCounterSlot);
+        execute.push(0);
+        execute.ifICmp(GeneratorAdapter.GT, end);
+        execute.throwException(PAINLESS_ERROR_TYPE,
+            "The maximum number of statements that can be executed in a loop has been reached.");
+        execute.mark(end);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterUtility.java
new file mode 100644
index 00000000000..d7e4f4ce42c
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterUtility.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Definition.Type;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.painless.PainlessParser.ADD;
+import static org.elasticsearch.painless.PainlessParser.BWAND;
+import static org.elasticsearch.painless.PainlessParser.BWOR;
+import static org.elasticsearch.painless.PainlessParser.BWXOR;
+import static org.elasticsearch.painless.PainlessParser.DIV;
+import static org.elasticsearch.painless.PainlessParser.LSH;
+import static org.elasticsearch.painless.PainlessParser.MUL;
+import static org.elasticsearch.painless.PainlessParser.REM;
+import static org.elasticsearch.painless.PainlessParser.RSH;
+import static org.elasticsearch.painless.PainlessParser.SUB;
+import static org.elasticsearch.painless.PainlessParser.USH;
+import static org.elasticsearch.painless.WriterConstants.ADDEXACT_INT;
+import static org.elasticsearch.painless.WriterConstants.ADDEXACT_LONG;
+import static org.elasticsearch.painless.WriterConstants.ADDWOOVERLOW_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.ADDWOOVERLOW_FLOAT;
+import static org.elasticsearch.painless.WriterConstants.DEF_ADD_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_AND_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_DIV_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_LSH_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_MUL_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_OR_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_REM_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_RSH_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_SUB_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_USH_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_XOR_CALL;
+import static org.elasticsearch.painless.WriterConstants.DIVWOOVERLOW_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.DIVWOOVERLOW_FLOAT;
+import static org.elasticsearch.painless.WriterConstants.DIVWOOVERLOW_INT;
+import static org.elasticsearch.painless.WriterConstants.DIVWOOVERLOW_LONG;
+import static org.elasticsearch.painless.WriterConstants.MULEXACT_INT;
+import static org.elasticsearch.painless.WriterConstants.MULEXACT_LONG;
+import static org.elasticsearch.painless.WriterConstants.MULWOOVERLOW_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.MULWOOVERLOW_FLOAT;
+import static org.elasticsearch.painless.WriterConstants.REMWOOVERLOW_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.REMWOOVERLOW_FLOAT;
+import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_BOOLEAN;
+import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_CHAR;
+import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_FLOAT;
+import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_INT;
+import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_LONG;
+import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_OBJECT;
+import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_STRING;
+import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_CONSTRUCTOR;
+import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_TOSTRING;
+import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_TYPE;
+import static org.elasticsearch.painless.WriterConstants.SUBEXACT_INT;
+import static org.elasticsearch.painless.WriterConstants.SUBEXACT_LONG;
+import static org.elasticsearch.painless.WriterConstants.SUBWOOVERLOW_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.SUBWOOVERLOW_FLOAT;
+
+class WriterUtility {
+    static class Branch {
+        final ParserRuleContext source;
+
+        Label begin = null;
+        Label end = null;
+        Label tru = null;
+        Label fals = null;
+
+        private Branch(final ParserRuleContext source) {
+            this.source = source;
+        }
+    }
+
+    /**
+     * A utility method to output consistent error messages.
+     * @param ctx The ANTLR node the error occurred in.
+     * @return The error message with tacked on line number and character position.
+     */
+    static String error(final ParserRuleContext ctx) {
+        return "Writer Error [" + ctx.getStart().getLine() + ":" + ctx.getStart().getCharPositionInLine() + "]: ";
+    }
+
+    private final Definition definition;
+    private final CompilerSettings settings;
+
+    private final GeneratorAdapter execute;
+
+    private final Map<ParserRuleContext, Branch> branches = new HashMap<>();
+    private final Deque<Branch> jumps = new ArrayDeque<>();
+    private final Set<ParserRuleContext> strings = new HashSet<>();
+
+    WriterUtility(final Metadata metadata, final GeneratorAdapter execute) {
+        definition = metadata.definition;
+        settings = metadata.settings;
+
+        this.execute = execute;
+    }
+
+    Branch markBranch(final ParserRuleContext source, final ParserRuleContext... nodes) {
+        final Branch branch = new Branch(source);
+
+        for (final ParserRuleContext node : nodes) {
+            branches.put(node, branch);
+        }
+
+        return branch;
+    }
+
+    void copyBranch(final Branch branch, final ParserRuleContext... nodes) {
+        for (final ParserRuleContext node : nodes) {
+            branches.put(node, branch);
+        }
+    }
+
+    Branch getBranch(final ParserRuleContext source) {
+        return branches.get(source);
+    }
+
+    void checkWriteBranch(final ParserRuleContext source) {
+        final Branch branch = getBranch(source);
+
+        if (branch != null) {
+            if (branch.tru != null) {
+                execute.visitJumpInsn(Opcodes.IFNE, branch.tru);
+            } else if (branch.fals != null) {
+                execute.visitJumpInsn(Opcodes.IFEQ, branch.fals);
+            }
+        }
+    }
+
+    void pushJump(final Branch branch) {
+        jumps.push(branch);
+    }
+
+    Branch peekJump() {
+        return jumps.peek();
+    }
+
+    void popJump() {
+        jumps.pop();
+    }
+
+    void addStrings(final ParserRuleContext source) {
+        strings.add(source);
+    }
+
+    boolean containsStrings(final ParserRuleContext source) {
+        return strings.contains(source);
+    }
+
+    void removeStrings(final ParserRuleContext source) {
+        strings.remove(source);
+    }
+
+    void writeDup(final int size, final boolean x1, final boolean x2) {
+        if (size == 1) {
+            if (x2) {
+                execute.dupX2();
+            } else if (x1) {
+                execute.dupX1();
+            } else {
+                execute.dup();
+            }
+        } else if (size == 2) {
+            if (x2) {
+                execute.dup2X2();
+            } else if (x1) {
+                execute.dup2X1();
+            } else {
+                execute.dup2();
+            }
+        }
+    }
+
+    void writePop(final int size) {
+        if (size == 1) {
+            execute.pop();
+        } else if (size == 2) {
+            execute.pop2();
+        }
+    }
+
+    void writeConstant(final ParserRuleContext source, final Object constant) {
+        if (constant instanceof Number) {
+            writeNumeric(source, constant);
+        } else if (constant instanceof Character) {
+            writeNumeric(source, (int)(char)constant);
+        } else if (constant instanceof String) {
+            writeString(source, constant);
+        } else if (constant instanceof Boolean) {
+            writeBoolean(source, constant);
+        } else if (constant != null) {
+            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+        }
+    }
+
+    void writeNumeric(final ParserRuleContext source, final Object numeric) {
+        if (numeric instanceof Double) {
+            execute.push((double)numeric);
+        } else if (numeric instanceof Float) {
+            execute.push((float)numeric);
+        } else if (numeric instanceof Long) {
+            execute.push((long)numeric);
+        } else if (numeric instanceof Number) {
+            execute.push(((Number)numeric).intValue());
+        } else {
+            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+        }
+    }
+
+    void writeString(final ParserRuleContext source, final Object string) {
+        if (string instanceof String) {
+            execute.push((String)string);
+        } else {
+            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+        }
+    }
+
+    void writeBoolean(final ParserRuleContext source, final Object bool) {
+        if (bool instanceof Boolean) {
+            execute.push((boolean)bool);
+        } else {
+            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+        }
+    }
+
+    void writeNewStrings() {
+        execute.newInstance(STRINGBUILDER_TYPE);
+        execute.dup();
+        execute.invokeConstructor(STRINGBUILDER_TYPE, STRINGBUILDER_CONSTRUCTOR);
+    }
+
+    void writeAppendStrings(final Sort sort) {
+        switch (sort) {
+            case BOOL:   execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_BOOLEAN); break;
+            case CHAR:   execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_CHAR);    break;
+            case BYTE:
+            case SHORT:
+            case INT:    execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_INT);     break;
+            case LONG:   execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_LONG);    break;
+            case FLOAT:  execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_FLOAT);   break;
+            case DOUBLE: execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_DOUBLE);  break;
+            case STRING: execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_STRING);  break;
+            default:     execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_OBJECT);
+        }
+    }
+
+    void writeToStrings() {
+        execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_TOSTRING);
+    }
+
+    void writeBinaryInstruction(final ParserRuleContext source, final Type type, final int token) {
+        final Sort sort = type.sort;
+        final boolean exact = !settings.getNumericOverflow() &&
+            ((sort == Sort.INT || sort == Sort.LONG) &&
+                (token == MUL || token == DIV || token == ADD || token == SUB) ||
+                (sort == Sort.FLOAT || sort == Sort.DOUBLE) &&
+                    (token == MUL || token == DIV || token == REM || token == ADD || token == SUB));
+
+        // If it's a 64-bit shift, fix-up the last argument to truncate to 32-bits.
+        // Note that unlike java, this means we still do binary promotion of shifts,
+        // but it keeps things simple, and this check works because we promote shifts.
+        if (sort == Sort.LONG && (token == LSH || token == USH || token == RSH)) {
+            execute.cast(org.objectweb.asm.Type.LONG_TYPE, org.objectweb.asm.Type.INT_TYPE);
+        }
+
+        if (exact) {
+            switch (sort) {
+                case INT:
+                    switch (token) {
+                        case MUL: execute.invokeStatic(definition.mathType.type,    MULEXACT_INT);     break;
+                        case DIV: execute.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_INT); break;
+                        case ADD: execute.invokeStatic(definition.mathType.type,    ADDEXACT_INT);     break;
+                        case SUB: execute.invokeStatic(definition.mathType.type,    SUBEXACT_INT);     break;
+                        default:
+                            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+                    }
+
+                    break;
+                case LONG:
+                    switch (token) {
+                        case MUL: execute.invokeStatic(definition.mathType.type,    MULEXACT_LONG);     break;
+                        case DIV: execute.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_LONG); break;
+                        case ADD: execute.invokeStatic(definition.mathType.type,    ADDEXACT_LONG);     break;
+                        case SUB: execute.invokeStatic(definition.mathType.type,    SUBEXACT_LONG);     break;
+                        default:
+                            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+                    }
+
+                    break;
+                case FLOAT:
+                    switch (token) {
+                        case MUL: execute.invokeStatic(definition.utilityType.type, MULWOOVERLOW_FLOAT); break;
+                        case DIV: execute.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_FLOAT); break;
+                        case REM: execute.invokeStatic(definition.utilityType.type, REMWOOVERLOW_FLOAT); break;
+                        case ADD: execute.invokeStatic(definition.utilityType.type, ADDWOOVERLOW_FLOAT); break;
+                        case SUB: execute.invokeStatic(definition.utilityType.type, SUBWOOVERLOW_FLOAT); break;
+                        default:
+                            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+                    }
+
+                    break;
+                case DOUBLE:
+                    switch (token) {
+                        case MUL: execute.invokeStatic(definition.utilityType.type, MULWOOVERLOW_DOUBLE); break;
+                        case DIV: execute.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_DOUBLE); break;
+                        case REM: execute.invokeStatic(definition.utilityType.type, REMWOOVERLOW_DOUBLE); break;
+                        case ADD: execute.invokeStatic(definition.utilityType.type, ADDWOOVERLOW_DOUBLE); break;
+                        case SUB: execute.invokeStatic(definition.utilityType.type, SUBWOOVERLOW_DOUBLE); break;
+                        default:
+                            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+                    }
+
+                    break;
+                default:
+                    throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+            }
+        } else {
+            if ((sort == Sort.FLOAT || sort == Sort.DOUBLE) &&
+                (token == LSH || token == USH || token == RSH || token == BWAND || token == BWXOR || token == BWOR)) {
+                throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+            }
+
+            if (sort == Sort.DEF) {
+                switch (token) {
+                    case MUL:   execute.invokeStatic(definition.defobjType.type, DEF_MUL_CALL); break;
+                    case DIV:   execute.invokeStatic(definition.defobjType.type, DEF_DIV_CALL); break;
+                    case REM:   execute.invokeStatic(definition.defobjType.type, DEF_REM_CALL); break;
+                    case ADD:   execute.invokeStatic(definition.defobjType.type, DEF_ADD_CALL); break;
+                    case SUB:   execute.invokeStatic(definition.defobjType.type, DEF_SUB_CALL); break;
+                    case LSH:   execute.invokeStatic(definition.defobjType.type, DEF_LSH_CALL); break;
+                    case USH:   execute.invokeStatic(definition.defobjType.type, DEF_RSH_CALL); break;
+                    case RSH:   execute.invokeStatic(definition.defobjType.type, DEF_USH_CALL); break;
+                    case BWAND: execute.invokeStatic(definition.defobjType.type, DEF_AND_CALL); break;
+                    case BWXOR: execute.invokeStatic(definition.defobjType.type, DEF_XOR_CALL); break;
+                    case BWOR:  execute.invokeStatic(definition.defobjType.type, DEF_OR_CALL);  break;
+                    default:
+                        throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+                }
+            } else {
+                switch (token) {
+                    case MUL:   execute.math(GeneratorAdapter.MUL,  type.type); break;
+                    case DIV:   execute.math(GeneratorAdapter.DIV,  type.type); break;
+                    case REM:   execute.math(GeneratorAdapter.REM,  type.type); break;
+                    case ADD:   execute.math(GeneratorAdapter.ADD,  type.type); break;
+                    case SUB:   execute.math(GeneratorAdapter.SUB,  type.type); break;
+                    case LSH:   execute.math(GeneratorAdapter.SHL,  type.type); break;
+                    case USH:   execute.math(GeneratorAdapter.USHR, type.type); break;
+                    case RSH:   execute.math(GeneratorAdapter.SHR,  type.type); break;
+                    case BWAND: execute.math(GeneratorAdapter.AND,  type.type); break;
+                    case BWXOR: execute.math(GeneratorAdapter.XOR,  type.type); break;
+                    case BWOR:  execute.math(GeneratorAdapter.OR,   type.type); break;
+                    default:
+                        throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+                }
+            }
+        }
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
index 861c03cd706..84b7ab58ef2 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
@@ -47,12 +47,14 @@ import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -62,13 +64,14 @@ import static java.util.Collections.emptyList;
 import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.action.bulk.BackoffPolicy.exponentialBackoff;
 import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
+import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.elasticsearch.index.reindex.AbstractBulkByScrollRequest.SIZE_ALL_MATCHES;
 import static org.elasticsearch.rest.RestStatus.CONFLICT;
 import static org.elasticsearch.search.sort.SortBuilders.fieldSort;
 
 /**
  * Abstract base for scrolling across a search and executing bulk actions on all
- * results.
+ * results. All package private methods are package private so their tests can use them.
  */
 public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBulkByScrollRequest<Request>, Response> {
     /**
@@ -80,6 +83,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
 
     private final AtomicLong startTime = new AtomicLong(-1);
     private final AtomicReference<String> scroll = new AtomicReference<>();
+    private final AtomicLong lastBatchStartTime = new AtomicLong(-1);
     private final Set<String> destinationIndices = Collections.newSetFromMap(new ConcurrentHashMap<>());
 
     private final ESLogger logger;
@@ -103,17 +107,13 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
 
     protected abstract BulkRequest buildBulk(Iterable<SearchHit> docs);
 
-    protected abstract Response buildResponse(TimeValue took, List<Failure> indexingFailures, List<ShardSearchFailure> searchFailures);
+    protected abstract Response buildResponse(TimeValue took, List<Failure> indexingFailures, List<ShardSearchFailure> searchFailures,
+            boolean timedOut);
 
+    /**
+     * Start the action by firing the initial search request.
+     */
     public void start() {
-        initialSearch();
-    }
-
-    public BulkByScrollTask getTask() {
-        return task;
-    }
-
-    void initialSearch() {
         if (task.isCancelled()) {
             finishHim(null);
             return;
@@ -135,7 +135,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
                 @Override
                 public void onResponse(SearchResponse response) {
                     logger.debug("[{}] documents match query", response.getHits().getTotalHits());
-                    onScrollResponse(response);
+                    onScrollResponse(timeValueSeconds(0), response);
                 }
 
                 @Override
@@ -149,20 +149,23 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
     }
 
     /**
-     * Set the last returned scrollId. Package private for testing.
+     * Process a scroll response.
+     * @param delay how long to delay processesing the response. This delay is how throttling is applied to the action.
+     * @param searchResponse the scroll response to process
      */
-    void setScroll(String scroll) {
-        this.scroll.set(scroll);
-    }
-
-    void onScrollResponse(SearchResponse searchResponse) {
+    void onScrollResponse(TimeValue delay, SearchResponse searchResponse) {
         if (task.isCancelled()) {
             finishHim(null);
             return;
         }
         setScroll(searchResponse.getScrollId());
-        if (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0) {
-            startNormalTermination(emptyList(), unmodifiableList(Arrays.asList(searchResponse.getShardFailures())));
+        if (    // If any of the shards failed that should abort the request.
+                (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0)
+                // Timeouts aren't shard failures but we still need to pass them back to the user.
+                || searchResponse.isTimedOut()
+                ) {
+            startNormalTermination(emptyList(), unmodifiableList(Arrays.asList(searchResponse.getShardFailures())),
+                    searchResponse.isTimedOut());
             return;
         }
         long total = searchResponse.getHits().totalHits();
@@ -170,13 +173,19 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
             total = min(total, mainRequest.getSize());
         }
         task.setTotal(total);
-        threadPool.generic().execute(new AbstractRunnable() {
+        task.countThrottle(delay);
+        threadPool.schedule(delay, ThreadPool.Names.GENERIC, threadPool.getThreadContext().preserveContext(new AbstractRunnable() {
             @Override
             protected void doRun() throws Exception {
+                if (task.isCancelled()) {
+                    finishHim(null);
+                    return;
+                }
+                lastBatchStartTime.set(System.nanoTime());
                 SearchHit[] docs = searchResponse.getHits().getHits();
                 logger.debug("scroll returned [{}] documents with a scroll id of [{}]", docs.length, searchResponse.getScrollId());
                 if (docs.length == 0) {
-                    startNormalTermination(emptyList(), emptyList());
+                    startNormalTermination(emptyList(), emptyList(), false);
                     return;
                 }
                 task.countBatch();
@@ -193,7 +202,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
                     /*
                      * If we noop-ed the entire batch then just skip to the next batch or the BulkRequest would fail validation.
                      */
-                    startNextScroll();
+                    startNextScroll(0);
                     return;
                 }
                 request.timeout(mainRequest.getTimeout());
@@ -209,9 +218,12 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
             public void onFailure(Throwable t) {
                 finishHim(t);
             }
-        });
+        }));
     }
 
+    /**
+     * Send a bulk request, handling retries.
+     */
     void sendBulkRequest(BulkRequest request) {
         if (task.isCancelled()) {
             finishHim(null);
@@ -230,6 +242,9 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
         });
     }
 
+    /**
+     * Processes bulk responses, accounting for failures.
+     */
     void onBulkResponse(BulkResponse response) {
         if (task.isCancelled()) {
             finishHim(null);
@@ -261,37 +276,46 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
                     throw new IllegalArgumentException("Unknown op type:  " + item.getOpType());
                 }
                 // Track the indexes we've seen so we can refresh them if requested
-                destinationIndices.add(item.getIndex());
+                destinationIndicesThisBatch.add(item.getIndex());
             }
-            destinationIndices.addAll(destinationIndicesThisBatch);
+            addDestinationIndices(destinationIndicesThisBatch);
 
             if (false == failures.isEmpty()) {
-                startNormalTermination(unmodifiableList(failures), emptyList());
+                startNormalTermination(unmodifiableList(failures), emptyList(), false);
                 return;
             }
 
             if (mainRequest.getSize() != SIZE_ALL_MATCHES && task.getSuccessfullyProcessed() >= mainRequest.getSize()) {
                 // We've processed all the requested docs.
-                startNormalTermination(emptyList(), emptyList());
+                startNormalTermination(emptyList(), emptyList(), false);
                 return;
             }
-            startNextScroll();
+            startNextScroll(response.getItems().length);
         } catch (Throwable t) {
             finishHim(t);
         }
     }
 
-    void startNextScroll() {
+    /**
+     * Start the next scroll request.
+     *
+     * @param lastBatchSize the number of requests sent in the last batch. This is used to calculate the throttling values which are applied
+     *        when the scroll returns
+     */
+    void startNextScroll(int lastBatchSize) {
         if (task.isCancelled()) {
             finishHim(null);
             return;
         }
+        long earliestNextBatchStartTime = lastBatchStartTime.get() + (long) perfectlyThrottledBatchTime(lastBatchSize);
+        long waitTime = max(0, earliestNextBatchStartTime - System.nanoTime());
         SearchScrollRequest request = new SearchScrollRequest();
-        request.scrollId(scroll.get()).scroll(firstSearchRequest.scroll());
+        // Add the wait time into the scroll timeout so it won't timeout while we wait for throttling
+        request.scrollId(scroll.get()).scroll(timeValueNanos(firstSearchRequest.scroll().keepAlive().nanos() + waitTime));
         client.searchScroll(request, new ActionListener<SearchResponse>() {
             @Override
             public void onResponse(SearchResponse response) {
-                onScrollResponse(response);
+                onScrollResponse(timeValueNanos(max(0, earliestNextBatchStartTime - System.nanoTime())), response);
             }
 
             @Override
@@ -301,6 +325,21 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
         });
     }
 
+    /**
+     * How many nanoseconds should a batch of lastBatchSize have taken if it were perfectly throttled? Package private for testing.
+     */
+    float perfectlyThrottledBatchTime(int lastBatchSize) {
+        if (mainRequest.getRequestsPerSecond() == 0) {
+            return 0;
+        }
+        //       requests
+        // ------------------- == seconds
+        // request per seconds
+        float targetBatchTimeInSeconds = lastBatchSize / mainRequest.getRequestsPerSecond();
+        // nanoseconds per seconds * seconds == nanoseconds
+        return TimeUnit.SECONDS.toNanos(1) * targetBatchTimeInSeconds;
+    }
+
     private void recordFailure(Failure failure, List<Failure> failures) {
         if (failure.getStatus() == CONFLICT) {
             task.countVersionConflict();
@@ -311,9 +350,12 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
         failures.add(failure);
     }
 
-    void startNormalTermination(List<Failure> indexingFailures, List<ShardSearchFailure> searchFailures) {
-        if (false == mainRequest.isRefresh()) {
-            finishHim(null, indexingFailures, searchFailures);
+    /**
+     * Start terminating a request that finished non-catastrophically.
+     */
+    void startNormalTermination(List<Failure> indexingFailures, List<ShardSearchFailure> searchFailures, boolean timedOut) {
+        if (task.isCancelled() || false == mainRequest.isRefresh() || destinationIndices.isEmpty()) {
+            finishHim(null, indexingFailures, searchFailures, timedOut);
             return;
         }
         RefreshRequest refresh = new RefreshRequest();
@@ -321,7 +363,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
         client.admin().indices().refresh(refresh, new ActionListener<RefreshResponse>() {
             @Override
             public void onResponse(RefreshResponse response) {
-                finishHim(null, indexingFailures, searchFailures);
+                finishHim(null, indexingFailures, searchFailures, timedOut);
             }
 
             @Override
@@ -337,7 +379,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
      * @param failure if non null then the request failed catastrophically with this exception
      */
     void finishHim(Throwable failure) {
-        finishHim(failure, emptyList(), emptyList());
+        finishHim(failure, emptyList(), emptyList(), false);
     }
 
     /**
@@ -346,8 +388,9 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
      * @param failure if non null then the request failed catastrophically with this exception
      * @param indexingFailures any indexing failures accumulated during the request
      * @param searchFailures any search failures accumulated during the request
+     * @param timedOut have any of the sub-requests timed out?
      */
-    void finishHim(Throwable failure, List<Failure> indexingFailures, List<ShardSearchFailure> searchFailures) {
+    void finishHim(Throwable failure, List<Failure> indexingFailures, List<ShardSearchFailure> searchFailures, boolean timedOut) {
         String scrollId = scroll.get();
         if (Strings.hasLength(scrollId)) {
             /*
@@ -369,19 +412,49 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
             });
         }
         if (failure == null) {
-            listener.onResponse(buildResponse(timeValueNanos(System.nanoTime() - startTime.get()), indexingFailures, searchFailures));
+            listener.onResponse(
+                    buildResponse(timeValueNanos(System.nanoTime() - startTime.get()), indexingFailures, searchFailures, timedOut));
         } else {
             listener.onFailure(failure);
         }
     }
 
     /**
-     * Build the backoff policy for use with retries. Package private for testing.
+     * Build the backoff policy for use with retries.
      */
     BackoffPolicy backoffPolicy() {
         return exponentialBackoff(mainRequest.getRetryBackoffInitialTime(), mainRequest.getMaxRetries());
     }
 
+    /**
+     * Add to the list of indices that were modified by this request. This is the list of indices refreshed at the end of the request if the
+     * request asks for a refresh.
+     */
+    void addDestinationIndices(Collection<String> indices) {
+        destinationIndices.addAll(indices);
+    }
+
+    /**
+     * Set the last returned scrollId. Exists entirely for testing.
+     */
+    void setScroll(String scroll) {
+        this.scroll.set(scroll);
+    }
+
+    /**
+     * Set the last batch's start time. Exists entirely for testing.
+     */
+    void setLastBatchStartTime(long newValue) {
+        lastBatchStartTime.set(newValue);
+    }
+
+    /**
+     * Get the last batch's start time. Exists entirely for testing.
+     */
+    long getLastBatchStartTime() {
+        return lastBatchStartTime.get();
+    }
+
     /**
      * Wraps a backoffPolicy in another policy that counts the number of backoffs acquired.
      */
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
index 6f50b216c9b..de792f2d916 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
@@ -19,11 +19,10 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
@@ -33,28 +32,36 @@ import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.search.suggest.Suggesters;
 import org.elasticsearch.tasks.LoggingTaskListener;
 import org.elasticsearch.tasks.Task;
 
 import java.io.IOException;
 
-public abstract class AbstractBaseReindexRestHandler<Request extends ActionRequest<Request>, Response extends BulkIndexByScrollResponse,
-        TA extends TransportAction<Request, Response>> extends BaseRestHandler {
+public abstract class AbstractBaseReindexRestHandler<
+                Request extends AbstractBulkByScrollRequest<Request>,
+                Response extends BulkIndexByScrollResponse,
+                TA extends TransportAction<Request, Response>
+            > extends BaseRestHandler {
     protected final IndicesQueriesRegistry indicesQueriesRegistry;
     protected final AggregatorParsers aggParsers;
+    protected final Suggesters suggesters;
     private final ClusterService clusterService;
     private final TA action;
 
     protected AbstractBaseReindexRestHandler(Settings settings, Client client,
-            IndicesQueriesRegistry indicesQueriesRegistry, AggregatorParsers aggParsers, ClusterService clusterService, TA action) {
+            IndicesQueriesRegistry indicesQueriesRegistry, AggregatorParsers aggParsers, Suggesters suggesters,
+            ClusterService clusterService, TA action) {
         super(settings, client);
         this.indicesQueriesRegistry = indicesQueriesRegistry;
         this.aggParsers = aggParsers;
+        this.suggesters = suggesters;
         this.clusterService = clusterService;
         this.action = action;
     }
 
     protected void execute(RestRequest request, Request internalRequest, RestChannel channel) throws IOException {
+        internalRequest.setRequestsPerSecond(request.paramAsFloat("requests_per_second", internalRequest.getRequestsPerSecond()));
         if (request.paramAsBoolean("wait_for_completion", true)) {
             action.execute(internalRequest, new BulkIndexByScrollResponseContentListener<Response>(channel));
             return;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
index 41b436e6074..29e2acb352e 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
@@ -85,6 +85,13 @@ public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScr
      */
     private int maxRetries = 11;
 
+    /**
+     * The throttle for this request in sub-requests per second. 0 means set no throttle and that is the default. Throttling is done between
+     * batches, as we start the next scroll requests. That way we can increase the scroll's timeout to make sure that it contains any time
+     * that we might wait.
+     */
+    private float requestsPerSecond = 0;
+
     public AbstractBulkByScrollRequest() {
     }
 
@@ -252,6 +259,21 @@ public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScr
         return self();
     }
 
+    /**
+     * The throttle for this request in sub-requests per second. 0 means set no throttle and that is the default.
+     */
+    public float getRequestsPerSecond() {
+        return requestsPerSecond;
+    }
+
+    /**
+     * Set the throttle for this request in sub-requests per second. 0 means set no throttle and that is the default.
+     */
+    public Self setRequestsPerSecond(float requestsPerSecond) {
+        this.requestsPerSecond = requestsPerSecond;
+        return self();
+    }
+
     @Override
     public Task createTask(long id, String type, String action) {
         return new BulkByScrollTask(id, type, action, getDescription());
@@ -269,6 +291,7 @@ public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScr
         consistency = WriteConsistencyLevel.fromId(in.readByte());
         retryBackoffInitialTime = TimeValue.readTimeValue(in);
         maxRetries = in.readVInt();
+        requestsPerSecond = in.readFloat();
     }
 
     @Override
@@ -282,6 +305,7 @@ public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScr
         out.writeByte(consistency.id());
         retryBackoffInitialTime.writeTo(out);
         out.writeVInt(maxRetries);
+        out.writeFloat(requestsPerSecond);
     }
 
     /**
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
index c9d8e3f188c..079dca2d4f2 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.reindex;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
@@ -30,6 +31,8 @@ import java.io.IOException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
+import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
+
 /**
  * Task storing information about a currently running BulkByScroll request.
  */
@@ -46,6 +49,7 @@ public class BulkByScrollTask extends CancellableTask {
     private final AtomicInteger batch = new AtomicInteger(0);
     private final AtomicLong versionConflicts = new AtomicLong(0);
     private final AtomicLong retries = new AtomicLong(0);
+    private final AtomicLong throttledNanos = new AtomicLong();
 
     public BulkByScrollTask(long id, String type, String action, String description) {
         super(id, type, action, description);
@@ -54,7 +58,7 @@ public class BulkByScrollTask extends CancellableTask {
     @Override
     public Status getStatus() {
         return new Status(total.get(), updated.get(), created.get(), deleted.get(), batch.get(), versionConflicts.get(), noops.get(),
-                retries.get(), getReasonCancelled());
+                retries.get(), timeValueNanos(throttledNanos.get()), getReasonCancelled());
     }
 
     /**
@@ -65,8 +69,7 @@ public class BulkByScrollTask extends CancellableTask {
     }
 
     public static class Status implements Task.Status {
-        public static final Status PROTOTYPE = new Status(0, 0, 0, 0, 0, 0, 0, 0, null);
-
+        public static final String NAME = "bulk-by-scroll";
         private final long total;
         private final long updated;
         private final long created;
@@ -75,10 +78,11 @@ public class BulkByScrollTask extends CancellableTask {
         private final long versionConflicts;
         private final long noops;
         private final long retries;
+        private final TimeValue throttled;
         private final String reasonCancelled;
 
         public Status(long total, long updated, long created, long deleted, int batches, long versionConflicts, long noops, long retries,
-                @Nullable String reasonCancelled) {
+                TimeValue throttled, @Nullable String reasonCancelled) {
             this.total = checkPositive(total, "total");
             this.updated = checkPositive(updated, "updated");
             this.created = checkPositive(created, "created");
@@ -87,6 +91,7 @@ public class BulkByScrollTask extends CancellableTask {
             this.versionConflicts = checkPositive(versionConflicts, "versionConflicts");
             this.noops = checkPositive(noops, "noops");
             this.retries = checkPositive(retries, "retries");
+            this.throttled = throttled;
             this.reasonCancelled = reasonCancelled;
         }
 
@@ -99,6 +104,7 @@ public class BulkByScrollTask extends CancellableTask {
             versionConflicts = in.readVLong();
             noops = in.readVLong();
             retries = in.readVLong();
+            throttled = TimeValue.readTimeValue(in);
             reasonCancelled = in.readOptionalString();
         }
 
@@ -112,6 +118,7 @@ public class BulkByScrollTask extends CancellableTask {
             out.writeVLong(versionConflicts);
             out.writeVLong(noops);
             out.writeVLong(retries);
+            throttled.writeTo(out);
             out.writeOptionalString(reasonCancelled);
         }
 
@@ -136,6 +143,7 @@ public class BulkByScrollTask extends CancellableTask {
             builder.field("version_conflicts", versionConflicts);
             builder.field("noops", noops);
             builder.field("retries", retries);
+            builder.timeValueField("throttled_millis", "throttled", throttled);
             if (reasonCancelled != null) {
                 builder.field("canceled", reasonCancelled);
             }
@@ -169,12 +177,7 @@ public class BulkByScrollTask extends CancellableTask {
 
         @Override
         public String getWriteableName() {
-            return "bulk-by-scroll";
-        }
-
-        @Override
-        public Status readFrom(StreamInput in) throws IOException {
-            return new Status(in);
+            return NAME;
         }
 
         /**
@@ -234,6 +237,13 @@ public class BulkByScrollTask extends CancellableTask {
             return retries;
         }
 
+        /**
+         * The total time this request has throttled itself.
+         */
+        public TimeValue getThrottled() {
+            return throttled;
+        }
+
         /**
          * The reason that the request was canceled or null if it hasn't been.
          */
@@ -287,4 +297,11 @@ public class BulkByScrollTask extends CancellableTask {
     void countRetry() {
         retries.incrementAndGet();
     }
+
+    public void countThrottle(TimeValue delay) {
+        long nanos = delay.nanos();
+        if (nanos > 0) {
+            throttledNanos.addAndGet(nanos);
+        }
+    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
index ca1a53ef999..60de9bfbd03 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
@@ -45,16 +45,18 @@ public class BulkIndexByScrollResponse extends ActionResponse implements ToXCont
     private BulkByScrollTask.Status status;
     private List<Failure> indexingFailures;
     private List<ShardSearchFailure> searchFailures;
+    private boolean timedOut;
 
     public BulkIndexByScrollResponse() {
     }
 
     public BulkIndexByScrollResponse(TimeValue took, BulkByScrollTask.Status status, List<Failure> indexingFailures,
-            List<ShardSearchFailure> searchFailures) {
+            List<ShardSearchFailure> searchFailures, boolean timedOut) {
         this.took = took;
         this.status = requireNonNull(status, "Null status not supported");
         this.indexingFailures = indexingFailures;
         this.searchFailures = searchFailures;
+        this.timedOut = timedOut;
     }
 
     public TimeValue getTook() {
@@ -103,6 +105,13 @@ public class BulkIndexByScrollResponse extends ActionResponse implements ToXCont
         return searchFailures;
     }
 
+    /**
+     * Did any of the sub-requests that were part of this request timeout?
+     */
+    public boolean isTimedOut() {
+        return timedOut;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
@@ -116,6 +125,7 @@ public class BulkIndexByScrollResponse extends ActionResponse implements ToXCont
         for (ShardSearchFailure failure: searchFailures) {
             failure.writeTo(out);
         }
+        out.writeBoolean(timedOut);
     }
 
     @Override
@@ -135,11 +145,13 @@ public class BulkIndexByScrollResponse extends ActionResponse implements ToXCont
             searchFailures.add(readShardSearchFailure(in));
         }
         this.searchFailures = unmodifiableList(searchFailures);
+        this.timedOut = in.readBoolean();
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("took", took.millis());
+        builder.field("timed_out", timedOut);
         status.innerXContent(builder, params, false, false);
         builder.startArray("failures");
         for (Failure failure: indexingFailures) {
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
index 24fdb16b397..24612aa14de 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
@@ -19,7 +19,9 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.support.RestToXContentListener;
@@ -35,12 +37,25 @@ public class BulkIndexByScrollResponseContentListener<R extends BulkIndexByScrol
 
     @Override
     protected RestStatus getStatus(R response) {
+        /*
+         * Return the highest numbered rest status under the assumption that higher numbered statuses are "more error" and thus more
+         * interesting to the user.
+         */
         RestStatus status = RestStatus.OK;
+        if (response.isTimedOut()) {
+            status = RestStatus.REQUEST_TIMEOUT;
+        }
         for (Failure failure : response.getIndexingFailures()) {
             if (failure.getStatus().getStatus() > status.getStatus()) {
                 status = failure.getStatus();
             }
         }
+        for (ShardSearchFailure failure: response.getSearchFailures()) {
+            RestStatus failureStatus = ExceptionsHelper.status(failure.getCause());
+            if (failureStatus.getStatus() > status.getStatus()) {
+                status = failureStatus;
+            }
+        }
         return status;
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
index a01c6e3b30e..9d84acf51c1 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
@@ -41,8 +41,9 @@ public class ReindexPlugin extends Plugin {
         actionModule.registerAction(UpdateByQueryAction.INSTANCE, TransportUpdateByQueryAction.class);
     }
 
-    public void onModule(NetworkModule restModule) {
-        restModule.registerRestHandler(RestReindexAction.class);
-        restModule.registerRestHandler(RestUpdateByQueryAction.class);
+    public void onModule(NetworkModule networkModule) {
+        networkModule.registerRestHandler(RestReindexAction.class);
+        networkModule.registerRestHandler(RestUpdateByQueryAction.class);
+        networkModule.registerTaskStatus(BulkByScrollTask.Status.NAME, BulkByScrollTask.Status::new);
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
index 1ac6117d02b..d51fb7e8bc1 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
@@ -19,19 +19,31 @@
 
 package org.elasticsearch.index.reindex;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.CompositeIndicesRequest;
+import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.uid.Versions;
 
-import java.io.IOException;
+import static java.util.Collections.unmodifiableList;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.index.VersionType.INTERNAL;
 
-public class ReindexRequest extends AbstractBulkIndexByScrollRequest<ReindexRequest> {
+/**
+ * Request to reindex some documents from one index to another. This implements CompositeIndicesRequest but in a misleading way. Rather than
+ * returning all the subrequests that it will make it tries to return a representative set of subrequests. This is best-effort for a bunch
+ * of reasons, not least of which that scripts are allowed to change the destination request in drastic ways, including changing the index
+ * to which documents are written.
+ */
+public class ReindexRequest extends AbstractBulkIndexByScrollRequest<ReindexRequest> implements CompositeIndicesRequest {
     /**
      * Prototype for index requests.
      */
@@ -123,4 +135,20 @@ public class ReindexRequest extends AbstractBulkIndexByScrollRequest<ReindexRequ
         }
         return b.toString();
     }
+
+    // CompositeIndicesRequest implementation so plugins can reason about the request. This is really just a best effort thing.
+    /**
+     * Accessor to get the underlying {@link IndicesRequest}s that this request wraps. Note that this method is <strong>not
+     * accurate</strong> since it returns a prototype {@link IndexRequest} and not the actual requests that will be issued as part of the
+     * execution of this request. Additionally, scripts can modify the underlying {@link IndexRequest} and change values such as the index,
+     * type, {@link org.elasticsearch.action.support.IndicesOptions}. In short - only use this for very course reasoning about the request.
+     *
+     * @return a list comprising of the {@link SearchRequest} and the prototype {@link IndexRequest}
+     */
+    @Override
+    public List<? extends IndicesRequest> subRequests() {
+        assert getSearchRequest() != null;
+        assert getDestination() != null;
+        return unmodifiableList(Arrays.asList(getSearchRequest(), getDestination()));
+    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexResponse.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexResponse.java
index a4aee0c00d3..7e74fe26ec2 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexResponse.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexResponse.java
@@ -35,8 +35,9 @@ public class ReindexResponse extends BulkIndexByScrollResponse {
     public ReindexResponse() {
     }
 
-    public ReindexResponse(TimeValue took, Status status, List<Failure> indexingFailures, List<ShardSearchFailure> searchFailures) {
-        super(took, status, indexingFailures, searchFailures);
+    public ReindexResponse(TimeValue took, Status status, List<Failure> indexingFailures, List<ShardSearchFailure> searchFailures,
+            boolean timedOut) {
+        super(took, status, indexingFailures, searchFailures, timedOut);
     }
 
     public long getCreated() {
@@ -46,6 +47,7 @@ public class ReindexResponse extends BulkIndexByScrollResponse {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("took", getTook());
+        builder.field("timed_out", isTimedOut());
         getStatus().innerXContent(builder, params, true, false);
         builder.startArray("failures");
         for (Failure failure: getIndexingFailures()) {
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
index 1130dc7beab..f78d4e282d3 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
@@ -23,7 +23,7 @@ import org.elasticsearch.action.WriteConsistencyLevel;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.inject.Inject;
@@ -43,6 +43,7 @@ import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.search.suggest.Suggesters;
 
 import java.io.IOException;
 import java.util.List;
@@ -76,7 +77,7 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
             builder.map(source);
             parser = parser.contentType().xContent().createParser(builder.bytes());
             context.queryParseContext.reset(parser);
-            search.source().parseXContent(parser, context.queryParseContext, context.aggParsers);
+            search.source().parseXContent(parser, context.queryParseContext, context.aggParsers, context.suggesters);
         };
 
         ObjectParser<IndexRequest, Void> destParser = new ObjectParser<>("dest");
@@ -102,9 +103,9 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
 
     @Inject
     public RestReindexAction(Settings settings, RestController controller, Client client,
-            IndicesQueriesRegistry indicesQueriesRegistry, AggregatorParsers aggParsers, ClusterService clusterService,
-            TransportReindexAction action) {
-        super(settings, client, indicesQueriesRegistry, aggParsers, clusterService, action);
+            IndicesQueriesRegistry indicesQueriesRegistry, AggregatorParsers aggParsers, Suggesters suggesters,
+            ClusterService clusterService, TransportReindexAction action) {
+        super(settings, client, indicesQueriesRegistry, aggParsers, suggesters, clusterService, action);
         controller.registerHandler(POST, "/_reindex", this);
     }
 
@@ -118,7 +119,8 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
         ReindexRequest internalRequest = new ReindexRequest(new SearchRequest(), new IndexRequest());
 
         try (XContentParser xcontent = XContentFactory.xContent(request.content()).createParser(request.content())) {
-            PARSER.parse(xcontent, internalRequest, new ReindexParseContext(new QueryParseContext(indicesQueriesRegistry), aggParsers));
+            PARSER.parse(xcontent, internalRequest, new ReindexParseContext(new QueryParseContext(indicesQueriesRegistry), aggParsers,
+                    suggesters));
         } catch (ParsingException e) {
             logger.warn("Bad request", e);
             badRequest(channel, e.getDetailedMessage());
@@ -170,10 +172,13 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
     private class ReindexParseContext {
         private final QueryParseContext queryParseContext;
         private final AggregatorParsers aggParsers;
+        private final Suggesters suggesters;
 
-        public ReindexParseContext(QueryParseContext queryParseContext, AggregatorParsers aggParsers) {
+        public ReindexParseContext(QueryParseContext queryParseContext, AggregatorParsers aggParsers,
+            Suggesters suggesters) {
             this.queryParseContext = queryParseContext;
             this.aggParsers = aggParsers;
+            this.suggesters = suggesters;
         }
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
index f4afd8c36e1..8da8b2c904f 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
@@ -38,6 +38,7 @@ import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.rest.action.support.RestActions;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.search.suggest.Suggesters;
 
 import java.util.Map;
 
@@ -49,9 +50,9 @@ public class RestUpdateByQueryAction extends
         AbstractBaseReindexRestHandler<UpdateByQueryRequest, BulkIndexByScrollResponse, TransportUpdateByQueryAction> {
     @Inject
     public RestUpdateByQueryAction(Settings settings, RestController controller, Client client,
-            IndicesQueriesRegistry indicesQueriesRegistry, AggregatorParsers aggParsers, ClusterService clusterService,
-            TransportUpdateByQueryAction action) {
-        super(settings, client, indicesQueriesRegistry, aggParsers, clusterService, action);
+            IndicesQueriesRegistry indicesQueriesRegistry, AggregatorParsers aggParsers, Suggesters suggesters,
+            ClusterService clusterService, TransportUpdateByQueryAction action) {
+        super(settings, client, indicesQueriesRegistry, aggParsers, suggesters, clusterService, action);
         controller.registerHandler(POST, "/{index}/_update_by_query", this);
         controller.registerHandler(POST, "/{index}/{type}/_update_by_query", this);
     }
@@ -96,7 +97,7 @@ public class RestUpdateByQueryAction extends
             }
         }
         RestSearchAction.parseSearchRequest(internalRequest.getSearchRequest(), indicesQueriesRegistry, request,
-                parseFieldMatcher, aggParsers, bodyContent);
+                parseFieldMatcher, aggParsers, suggesters, bodyContent);
 
         String conflicts = request.param("conflicts");
         if (conflicts != null) {
@@ -107,7 +108,10 @@ public class RestUpdateByQueryAction extends
         internalRequest.setSize(internalRequest.getSearchRequest().source().size());
         internalRequest.setPipeline(request.param("pipeline"));
         internalRequest.getSearchRequest().source().size(request.paramAsInt("scroll_size", scrollSize));
-
+        // Let the requester set search timeout. It is probably only going to be useful for testing but who knows.
+        if (request.hasParam("search_timeout")) {
+            internalRequest.getSearchRequest().source().timeout(request.paramAsTime("search_timeout", null));
+        }
 
         execute(request, internalRequest, channel);
     }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
index 069ee032f8e..482f101653d 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
@@ -29,9 +29,9 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.lucene.uid.Versions;
@@ -96,9 +96,9 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
              * it. This is the same sort of dance that TransportIndexRequest
              * uses to decide to autocreate the index.
              */
-            target = indexNameExpressionResolver.concreteIndices(clusterState, destination)[0];
+            target = indexNameExpressionResolver.concreteIndexNames(clusterState, destination)[0];
         }
-        for (String sourceIndex: indexNameExpressionResolver.concreteIndices(clusterState, source)) {
+        for (String sourceIndex: indexNameExpressionResolver.concreteIndexNames(clusterState, source)) {
             if (sourceIndex.equals(target)) {
                 ActionRequestValidationException e = new ActionRequestValidationException();
                 e.addValidationError("reindex cannot write into an index its reading from [" + target + ']');
@@ -191,8 +191,9 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
         }
 
         @Override
-        protected ReindexResponse buildResponse(TimeValue took, List<Failure> indexingFailures, List<ShardSearchFailure> searchFailures) {
-            return new ReindexResponse(took, task.getStatus(), indexingFailures, searchFailures);
+        protected ReindexResponse buildResponse(TimeValue took, List<Failure> indexingFailures, List<ShardSearchFailure> searchFailures,
+                boolean timedOut) {
+            return new ReindexResponse(took, task.getStatus(), indexingFailures, searchFailures, timedOut);
         }
 
         /*
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
index 0e13c6718dd..d004e86ac0c 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
@@ -96,8 +96,8 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
 
         @Override
         protected BulkIndexByScrollResponse buildResponse(TimeValue took, List<Failure> indexingFailures,
-                List<ShardSearchFailure> searchFailures) {
-            return new BulkIndexByScrollResponse(took, task.getStatus(), indexingFailures, searchFailures);
+                List<ShardSearchFailure> searchFailures, boolean timedOut) {
+            return new BulkIndexByScrollResponse(took, task.getStatus(), indexingFailures, searchFailures, timedOut);
         }
 
         @Override
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
index b2775393877..915921d6077 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
@@ -19,13 +19,23 @@
 
 package org.elasticsearch.index.reindex;
 
+import java.util.ArrayList;
+import java.util.List;
+
+import org.elasticsearch.action.CompositeIndicesRequest;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 
+import static java.util.Collections.unmodifiableList;
+
 /**
- * Request to reindex a set of documents where they are without changing their
- * locations or IDs.
+ * Request to update some documents. That means you can't change their type, id, index, or anything like that. This implements
+ * CompositeIndicesRequest but in a misleading way. Rather than returning all the subrequests that it will make it tries to return a
+ * representative set of subrequests. This is best-effort but better than {@linkplain ReindexRequest} because scripts can't change the
+ * destination index and things.
  */
-public class UpdateByQueryRequest extends AbstractBulkIndexByScrollRequest<UpdateByQueryRequest> {
+public class UpdateByQueryRequest extends AbstractBulkIndexByScrollRequest<UpdateByQueryRequest> implements CompositeIndicesRequest {
     /**
      * Ingest pipeline to set on index requests made by this action.
      */
@@ -64,4 +74,26 @@ public class UpdateByQueryRequest extends AbstractBulkIndexByScrollRequest<Updat
         searchToString(b);
         return b.toString();
     }
+
+    // CompositeIndicesRequest implementation so plugins can reason about the request. This is really just a best effort thing.
+    /**
+     * Accessor to get the underlying {@link IndicesRequest}s that this request wraps. Note that this method is <strong>not
+     * accurate</strong> since it returns dummy {@link IndexRequest}s and not the actual requests that will be issued as part of the
+     * execution of this request.
+     *
+     * @return a list comprising of the {@link SearchRequest} and dummy {@link IndexRequest}s
+     */
+    @Override
+    public List<? extends IndicesRequest> subRequests() {
+        assert getSearchRequest() != null;
+        List<IndicesRequest> subRequests = new ArrayList<>();
+        // One dummy IndexRequest per destination index.
+        for (String index : getSearchRequest().indices()) {
+            IndexRequest request = new IndexRequest();
+            request.index(index);
+            subRequests.add(request);
+        }
+        subRequests.add(getSearchRequest());
+        return unmodifiableList(subRequests);
+    };
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
index ae05f3270df..b734a555fa7 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.bulk.BackoffPolicy;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
@@ -38,6 +39,7 @@ import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.ClearScrollResponse;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.replication.ReplicationRequest;
@@ -66,24 +68,33 @@ import org.junit.After;
 import org.junit.Before;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
+import static java.util.Collections.singleton;
 import static org.apache.lucene.util.TestUtil.randomSimpleString;
 import static org.elasticsearch.action.bulk.BackoffPolicy.constantBackoff;
+import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
+import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.emptyCollectionOf;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -96,17 +107,24 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
     private String scrollId;
     private TaskManager taskManager;
     private BulkByScrollTask task;
+    private Map<String, String> expectedHeaders = new HashMap<>();
 
     @Before
     public void setupForTest() {
         client = new MyMockClient(new NoOpClient(getTestName()));
         threadPool = new ThreadPool(getTestName());
         mainRequest = new DummyAbstractBulkByScrollRequest();
-        firstSearchRequest = null;
+        firstSearchRequest = new SearchRequest().scroll(timeValueSeconds(10));
         listener = new PlainActionFuture<>();
         scrollId = null;
         taskManager = new TaskManager(Settings.EMPTY);
         task = (BulkByScrollTask) taskManager.register("don'tcare", "hereeither", mainRequest);
+
+        // Fill the context with something random so we can make sure we inherited it appropriately.
+        expectedHeaders.clear();
+        expectedHeaders.put(randomSimpleString(random()), randomSimpleString(random()));
+        threadPool.getThreadContext().newStoredContext();
+        threadPool.getThreadContext().putHeader(expectedHeaders);
     }
 
     @After
@@ -132,34 +150,40 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         long total = randomIntBetween(0, Integer.MAX_VALUE);
         InternalSearchHits hits = new InternalSearchHits(null, total, 0);
         InternalSearchResponse searchResponse = new InternalSearchResponse(hits, null, null, null, false, false);
-        new DummyAbstractAsyncBulkByScrollAction()
-            .onScrollResponse(new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
+        new DummyAbstractAsyncBulkByScrollAction().onScrollResponse(timeValueSeconds(0),
+                new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
         assertEquals(total, task.getStatus().getTotal());
     }
 
-    public void testEachScrollResponseIsABatch() {
-        // Replace the generic thread pool with one that executes immediately so the batch is updated immediately
-        threadPool.shutdown();
-        threadPool = new ThreadPool(getTestName()) {
-            @Override
-            public Executor generic() {
-                return new Executor() {
-                    @Override
-                    public void execute(Runnable command) {
-                        command.run();
-                    }
-                };
-            }
-        };
+    /**
+     * Tests that each scroll response is a batch and that the batch is launched properly.
+     */
+    public void testScrollResponseBatchingBehavior() throws Exception {
         int maxBatches = randomIntBetween(0, 100);
         for (int batches = 1; batches < maxBatches; batches++) {
+            long now = System.nanoTime();
             InternalSearchHit hit = new InternalSearchHit(0, "id", new Text("type"), emptyMap());
             InternalSearchHits hits = new InternalSearchHits(new InternalSearchHit[] { hit }, 0, 0);
             InternalSearchResponse searchResponse = new InternalSearchResponse(hits, null, null, null, false, false);
-            new DummyAbstractAsyncBulkByScrollAction()
-                .onScrollResponse(new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
+            DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
+            action.onScrollResponse(timeValueSeconds(0),
+                    new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
 
-            assertEquals(batches, task.getStatus().getBatches());
+            // Use assert busy because the update happens on another thread
+            final int expectedBatches = batches;
+            assertBusy(() -> assertEquals(expectedBatches, task.getStatus().getBatches()));
+
+            /*
+             * While we're here we can check that getting a scroll response sets the last scroll start time which makes sure the wait time
+             * isn't counted as time that the last batch took.
+             */
+            assertThat(action.getLastBatchStartTime(), greaterThanOrEqualTo(now));
+
+            /*
+             * Also while we're here check that we preserved the headers from the last request. assertBusy because no requests might have
+             * come in yet.
+             */
+            assertBusy(() -> assertEquals(expectedHeaders, client.lastHeaders.get()));
         }
     }
 
@@ -217,22 +241,20 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
      * Mimicks a ThreadPool rejecting execution of the task.
      */
     public void testThreadPoolRejectionsAbortRequest() throws Exception {
+        TimeValue expectedDelay = parseTimeValue(randomPositiveTimeValue(), "test");
         threadPool.shutdown();
         threadPool = new ThreadPool(getTestName()) {
             @Override
-            public Executor generic() {
-                return new Executor() {
-                    @Override
-                    public void execute(Runnable command) {
-                        ((AbstractRunnable) command).onRejection(new EsRejectedExecutionException("test"));
-                    }
-                };
+            public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
+                assertEquals(expectedDelay, delay); // While we're here we can check that the sleep made it through
+                ((AbstractRunnable) command).onRejection(new EsRejectedExecutionException("test"));
+                return null;
             }
         };
         InternalSearchHits hits = new InternalSearchHits(null, 0, 0);
         InternalSearchResponse searchResponse = new InternalSearchResponse(hits, null, null, null, false, false);
         new DummyAbstractAsyncBulkByScrollAction()
-                .onScrollResponse(new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
+                .onScrollResponse(expectedDelay, new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
         try {
             listener.get();
             fail("Expected a failure");
@@ -240,6 +262,9 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
             assertThat(e.getMessage(), equalTo("EsRejectedExecutionException[test]"));
         }
         assertThat(client.scrollsCleared, contains(scrollId));
+
+        // While we're mocking the threadPool lets also check that we incremented the throttle counter
+        assertEquals(expectedDelay, task.getStatus().getThrottled());
     }
 
     /**
@@ -248,15 +273,33 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
      */
     public void testShardFailuresAbortRequest() throws Exception {
         ShardSearchFailure shardFailure = new ShardSearchFailure(new RuntimeException("test"));
-        new DummyAbstractAsyncBulkByScrollAction()
-                .onScrollResponse(new SearchResponse(null, scrollId(), 5, 4, randomLong(), new ShardSearchFailure[] { shardFailure }));
+        InternalSearchResponse internalResponse = new InternalSearchResponse(null, null, null, null, false, null);
+        new DummyAbstractAsyncBulkByScrollAction().onScrollResponse(timeValueSeconds(0),
+                new SearchResponse(internalResponse, scrollId(), 5, 4, randomLong(), new ShardSearchFailure[] { shardFailure }));
         BulkIndexByScrollResponse response = listener.get();
         assertThat(response.getIndexingFailures(), emptyCollectionOf(Failure.class));
         assertThat(response.getSearchFailures(), contains(shardFailure));
+        assertFalse(response.isTimedOut());
         assertNull(response.getReasonCancelled());
         assertThat(client.scrollsCleared, contains(scrollId));
     }
 
+    /**
+     * Mimicks search timeouts.
+     */
+    public void testSearchTimeoutsAbortRequest() throws Exception {
+        InternalSearchResponse internalResponse = new InternalSearchResponse(null, null, null, null, true, null);
+        new DummyAbstractAsyncBulkByScrollAction().onScrollResponse(timeValueSeconds(0),
+                new SearchResponse(internalResponse, scrollId(), 5, 4, randomLong(), new ShardSearchFailure[0]));
+        BulkIndexByScrollResponse response = listener.get();
+        assertThat(response.getIndexingFailures(), emptyCollectionOf(Failure.class));
+        assertThat(response.getSearchFailures(), emptyCollectionOf(ShardSearchFailure.class));
+        assertTrue(response.isTimedOut());
+        assertNull(response.getReasonCancelled());
+        assertThat(client.scrollsCleared, contains(scrollId));
+    }
+
+
     /**
      * Mimicks bulk indexing failures.
      */
@@ -283,7 +326,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         InternalSearchHit hit = new InternalSearchHit(0, "id", new Text("type"), emptyMap());
         InternalSearchHits hits = new InternalSearchHits(new InternalSearchHit[] {hit}, 0, 0);
         InternalSearchResponse searchResponse = new InternalSearchResponse(hits, null, null, null, false, false);
-        action.onScrollResponse(new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
+        action.onScrollResponse(timeValueSeconds(0), new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
         try {
             listener.get();
             fail("Expected failure.");
@@ -313,6 +356,55 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         assertEquals(retryAttempts, task.getStatus().getRetries());
     }
 
+    public void testPerfectlyThrottledBatchTime() {
+        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
+        mainRequest.setRequestsPerSecond(0);
+        assertThat((double) action.perfectlyThrottledBatchTime(randomInt()), closeTo(0f, 0f));
+
+        int total = between(0, 1000000);
+        mainRequest.setRequestsPerSecond(1);
+        assertThat((double) action.perfectlyThrottledBatchTime(total),
+                closeTo(TimeUnit.SECONDS.toNanos(total), TimeUnit.SECONDS.toNanos(1)));
+    }
+
+    public void testScrollDelay() throws Exception {
+        /*
+         * Replace the thread pool with one that will save the delay sent for the command. We'll use that to check that we used a proper
+         * delay for throttling.
+         */
+        AtomicReference<TimeValue> capturedDelay = new AtomicReference<>();
+        threadPool.shutdown();
+        threadPool = new ThreadPool(getTestName()) {
+            @Override
+            public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
+                capturedDelay.set(delay);
+                return null;
+            }
+        };
+
+        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
+        action.setScroll(scrollId());
+
+        // We'd like to get about 1 request a second
+        mainRequest.setRequestsPerSecond(1f);
+        // Make the last scroll look nearly instant
+        action.setLastBatchStartTime(System.nanoTime());
+        // The last batch had 100 documents
+        action.startNextScroll(100);
+
+        // So the next request is going to have to wait an extra 100 seconds or so (base was 10, so 110ish)
+        assertThat(client.lastScroll.get().request.scroll().keepAlive().seconds(), either(equalTo(110L)).or(equalTo(109L)));
+
+        // Now we can simulate a response and check the delay that we used for the task
+        InternalSearchHit hit = new InternalSearchHit(0, "id", new Text("type"), emptyMap());
+        InternalSearchHits hits = new InternalSearchHits(new InternalSearchHit[] { hit }, 0, 0);
+        InternalSearchResponse searchResponse = new InternalSearchResponse(hits, null, null, null, false, false);
+        client.lastScroll.get().listener.onResponse(new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
+
+        // The delay is still 100ish seconds because there hasn't been much time between when we requested the bulk and when we got it.
+        assertThat(capturedDelay.get().seconds(), either(equalTo(100L)).or(equalTo(99L)));
+    }
+
     private long retryTestCase(boolean failWithRejection) throws Exception {
         int totalFailures = randomIntBetween(1, mainRequest.getMaxRetries());
         int size = randomIntBetween(1, 100);
@@ -332,7 +424,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
             }
 
             @Override
-            void startNextScroll() {
+            void startNextScroll(int lastBatchSize) {
                 successLatch.countDown();
             }
         };
@@ -370,13 +462,45 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         assertEquals(defaultBackoffBeforeFailing, millis);
     }
 
+    public void testRefreshIsFalseByDefault() throws Exception {
+        refreshTestCase(null, true, false);
+    }
+
+    public void testRefreshFalseDoesntExecuteRefresh() throws Exception {
+        refreshTestCase(false, true, false);
+    }
+
+    public void testRefreshTrueExecutesRefresh() throws Exception {
+        refreshTestCase(true, true, true);
+    }
+
+    public void testRefreshTrueSkipsRefreshIfNoDestinationIndexes() throws Exception {
+        refreshTestCase(true, false, false);
+    }
+
+    private void refreshTestCase(Boolean refresh, boolean addDestinationIndexes, boolean shouldRefresh) {
+        if (refresh != null) {
+            mainRequest.setRefresh(refresh);
+        }
+        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
+        if (addDestinationIndexes) {
+            action.addDestinationIndices(singleton("foo"));
+        }
+        action.startNormalTermination(emptyList(), emptyList(), false);
+        if (shouldRefresh) {
+            assertArrayEquals(new String[] {"foo"}, client.lastRefreshRequest.get().indices());
+        } else {
+            assertNull("No refresh was attempted", client.lastRefreshRequest.get());
+        }
+    }
+
     public void testCancelBeforeInitialSearch() throws Exception {
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.initialSearch());
+        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.start());
     }
 
     public void testCancelBeforeScrollResponse() throws Exception {
         // We bail so early we don't need to pass in a half way valid response.
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.onScrollResponse(null));
+        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.onScrollResponse(timeValueSeconds(0), null));
     }
 
     public void testCancelBeforeSendBulkRequest() throws Exception {
@@ -390,14 +514,54 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
     }
 
     public void testCancelBeforeStartNextScroll() throws Exception {
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.startNextScroll());
+        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.startNextScroll(0));
     }
 
     public void testCancelBeforeStartNormalTermination() throws Exception {
         // Refresh or not doesn't matter - we don't try to refresh.
         mainRequest.setRefresh(usually());
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.startNormalTermination(emptyList(), emptyList()));
-        // This wouldn't return if we called refresh - the action would hang waiting for the refresh that we haven't mocked.
+        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.startNormalTermination(emptyList(), emptyList(), false));
+        assertNull("No refresh was attempted", client.lastRefreshRequest.get());
+    }
+
+    /**
+     * Tests that we can cancel the request during its throttling delay. This can't use {@link #cancelTaskCase(Consumer)} because it needs
+     * to send the request un-canceled and cancel it at a specific time.
+     */
+    public void testCancelWhileDelayedAfterScrollResponse() throws Exception {
+        String reason = randomSimpleString(random());
+
+        /*
+         * Replace the thread pool with one that will cancel the task as soon as anything is scheduled, which reindex tries to do when there
+         * is a delay.
+         */
+        threadPool.shutdown();
+        threadPool = new ThreadPool(getTestName()) {
+            @Override
+            public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
+                taskManager.cancel(task, reason, (Set<String> s) -> {});
+                command.run();
+                return null;
+            }
+        };
+
+        // Send the scroll response which will trigger the custom thread pool above, canceling the request before running the response
+        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
+        boolean previousScrollSet = usually();
+        if (previousScrollSet) {
+            action.setScroll(scrollId());
+        }
+        long total = randomIntBetween(0, Integer.MAX_VALUE);
+        InternalSearchHits hits = new InternalSearchHits(null, total, 0);
+        InternalSearchResponse searchResponse = new InternalSearchResponse(hits, null, null, null, false, false);
+        action.onScrollResponse(timeValueSeconds(0), new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
+
+        // Now that we've got our cancel we'll just verify that it all came through allright
+        assertEquals(reason, listener.get().getReasonCancelled());
+        if (previousScrollSet) {
+            // Canceled tasks always start to clear the scroll before they die.
+            assertThat(client.scrollsCleared, contains(scrollId));
+        }
     }
 
     private void cancelTaskCase(Consumer<DummyAbstractAsyncBulkByScrollAction> testMe) throws Exception {
@@ -430,8 +594,8 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
 
         @Override
         protected BulkIndexByScrollResponse buildResponse(TimeValue took, List<Failure> indexingFailures,
-                List<ShardSearchFailure> searchFailures) {
-            return new BulkIndexByScrollResponse(took, task.getStatus(), indexingFailures, searchFailures);
+                List<ShardSearchFailure> searchFailures, boolean timedOut) {
+            return new BulkIndexByScrollResponse(took, task.getStatus(), indexingFailures, searchFailures, timedOut);
         }
     }
 
@@ -442,9 +606,12 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         }
     }
 
-    private static class MyMockClient extends FilterClient {
+    private class MyMockClient extends FilterClient {
         private final List<String> scrollsCleared = new ArrayList<>();
         private final AtomicInteger bulksAttempts = new AtomicInteger();
+        private final AtomicReference<Map<String, String>> lastHeaders = new AtomicReference<>();
+        private final AtomicReference<RefreshRequest> lastRefreshRequest = new AtomicReference<>();
+        private final AtomicReference<RequestAndListener<SearchScrollRequest, SearchResponse>> lastScroll = new AtomicReference<>();
 
         private int bulksToReject = 0;
 
@@ -457,6 +624,16 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         protected <Request extends ActionRequest<Request>, Response extends ActionResponse,
                 RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(
                 Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
+            lastHeaders.set(threadPool.getThreadContext().getHeaders());
+            if (request instanceof RefreshRequest) {
+                lastRefreshRequest.set((RefreshRequest) request);
+                listener.onResponse(null);
+                return;
+            }
+            if (request instanceof SearchScrollRequest) {
+                lastScroll.set(new RequestAndListener<>((SearchScrollRequest) request, (ActionListener<SearchResponse>) listener));
+                return;
+            }
             if (request instanceof ClearScrollRequest) {
                 ClearScrollRequest clearScroll = (ClearScrollRequest) request;
                 scrollsCleared.addAll(clearScroll.getScrollIds());
@@ -508,4 +685,14 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
             super.doExecute(action, request, listener);
         }
     }
+
+    private static class RequestAndListener<Request extends ActionRequest<Request>, Response> {
+        private final Request request;
+        private final ActionListener<Response> listener;
+
+        public RequestAndListener(Request request, ActionListener<Response> listener) {
+            this.request = request;
+            this.listener = listener;
+        }
+    }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
index 81a3a2cc706..442943be21f 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
@@ -19,9 +19,12 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
+import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
+
 public class BulkByScrollTaskTests extends ESTestCase {
     private BulkByScrollTask task;
 
@@ -101,13 +104,14 @@ public class BulkByScrollTaskTests extends ESTestCase {
     }
 
     public void testStatusHatesNegatives() {
-        expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(-1, 0, 0, 0, 0, 0, 0, 0, null));
-        expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(0, -1, 0, 0, 0, 0, 0, 0, null));
-        expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(0, 0, -1, 0, 0, 0, 0, 0, null));
-        expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(0, 0, 0, -1, 0, 0, 0, 0, null));
-        expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(0, 0, 0, 0, -1, 0, 0, 0, null));
-        expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(0, 0, 0, 0, 0, -1, 0, 0, null));
-        expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(0, 0, 0, 0, 0, 0, -1, 0, null));
-        expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(0, 0, 0, 0, 0, 0, 0, -1, null));
+        TimeValue throttle = parseTimeValue(randomPositiveTimeValue(), "test");
+        expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(-1, 0, 0, 0, 0, 0, 0, 0, throttle, null));
+        expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(0, -1, 0, 0, 0, 0, 0, 0, throttle, null));
+        expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(0, 0, -1, 0, 0, 0, 0, 0, throttle, null));
+        expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(0, 0, 0, -1, 0, 0, 0, 0, throttle, null));
+        expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(0, 0, 0, 0, -1, 0, 0, 0, throttle, null));
+        expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(0, 0, 0, 0, 0, -1, 0, 0, throttle, null));
+        expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(0, 0, 0, 0, 0, 0, -1, 0, throttle, null));
+        expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(0, 0, 0, 0, 0, 0, 0, -1, throttle, null));
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTestUtils.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTestUtils.java
index d1f6b1ee171..5117d2781be 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTestUtils.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTestUtils.java
@@ -21,7 +21,10 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ListenableActionFuture;
+import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
+import org.elasticsearch.index.reindex.BulkByScrollTask.Status;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.NativeScriptFactory;
@@ -41,7 +44,10 @@ import java.util.concurrent.TimeoutException;
 
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.test.ESIntegTestCase.client;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.hasSize;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThat;
 
 /**
@@ -76,10 +82,26 @@ public class CancelTestUtils {
         // Wait until the script is on the second document.
         barrier.await(30, TimeUnit.SECONDS);
 
+        // Status should show running
+        ListTasksResponse tasksList = client().admin().cluster().prepareListTasks().setActions(actionToCancel).setDetailed(true).get();
+        assertThat(tasksList.getNodeFailures(), empty());
+        assertThat(tasksList.getTaskFailures(), empty());
+        assertThat(tasksList.getTasks(), hasSize(1));
+        BulkByScrollTask.Status status = (Status) tasksList.getTasks().get(0).getStatus();
+        assertNull(status.getReasonCancelled());
+
         // Cancel the request while the script is running. This will prevent the request from being sent at all.
         List<TaskInfo> cancelledTasks = client().admin().cluster().prepareCancelTasks().setActions(actionToCancel).get().getTasks();
         assertThat(cancelledTasks, hasSize(1));
 
+        // The status should now show canceled. The request will still be in the list because the script is still blocked.
+        tasksList = client().admin().cluster().prepareListTasks().setActions(actionToCancel).setDetailed(true).get();
+        assertThat(tasksList.getNodeFailures(), empty());
+        assertThat(tasksList.getTaskFailures(), empty());
+        assertThat(tasksList.getTasks(), hasSize(1));
+        status = (Status) tasksList.getTasks().get(0).getStatus();
+        assertEquals(CancelTasksRequest.DEFAULT_REASON, status.getReasonCancelled());
+
         // Now let the next document through. It won't be sent because the request is cancelled but we need to unblock the script.
         barrier.await();
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
index 83dcd1483c1..c169f6819ea 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
@@ -19,14 +19,12 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 
 import java.util.ArrayList;
 import java.util.List;
 
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 
 public class ReindexBasicTests extends ReindexTestCase {
@@ -84,40 +82,4 @@ public class ReindexBasicTests extends ReindexTestCase {
         assertThat(copy.get(), responseMatcher().created(half).batches(half, 5));
         assertHitCount(client().prepareSearch("dest").setTypes("half").setSize(0).get(), half);
     }
-
-    public void testRefreshIsFalseByDefault() throws Exception {
-        refreshTestCase(null, false);
-    }
-
-    public void testRefreshFalseDoesntMakeVisible() throws Exception {
-        refreshTestCase(false, false);
-    }
-
-    public void testRefreshTrueMakesVisible() throws Exception {
-        refreshTestCase(true, true);
-    }
-
-    /**
-     * Executes a reindex into an index with -1 refresh_interval and checks that
-     * the documents are visible properly.
-     */
-    private void refreshTestCase(Boolean refresh, boolean visible) throws Exception {
-        CreateIndexRequestBuilder create = client().admin().indices().prepareCreate("dest").setSettings("refresh_interval", -1);
-        assertAcked(create);
-        ensureYellow();
-        indexRandom(true, client().prepareIndex("source", "test", "1").setSource("foo", "a"),
-                client().prepareIndex("source", "test", "2").setSource("foo", "a"),
-                client().prepareIndex("source", "test", "3").setSource("foo", "b"),
-                client().prepareIndex("source", "test", "4").setSource("foo", "c"));
-        assertHitCount(client().prepareSearch("source").setSize(0).get(), 4);
-
-        // Copy all the docs
-        ReindexRequestBuilder copy = reindex().source("source").destination("dest", "all");
-        if (refresh != null) {
-            copy.refresh(refresh);
-        }
-        assertThat(copy.get(), responseMatcher().created(4));
-
-        assertHitCount(client().prepareSearch("dest").setTypes("all").setSize(0).get(), visible ? 4 : 0);
-    }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
index f5c31fe8f42..cfa763e5dba 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
@@ -42,6 +42,7 @@ import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonList;
 import static org.apache.lucene.util.TestUtil.randomSimpleString;
+import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 
 /**
@@ -77,6 +78,7 @@ public class RoundTripTests extends ESTestCase {
         request.setTimeout(TimeValue.parseTimeValue(randomTimeValue(), null, "test"));
         request.setConsistency(randomFrom(WriteConsistencyLevel.values()));
         request.setScript(random().nextBoolean() ? null : randomScript());
+        request.setRequestsPerSecond(between(0, Integer.MAX_VALUE));
     }
 
     private void assertRequestEquals(AbstractBulkIndexByScrollRequest<?> request,
@@ -90,6 +92,7 @@ public class RoundTripTests extends ESTestCase {
         assertEquals(request.getScript(), tripped.getScript());
         assertEquals(request.getRetryBackoffInitialTime(), tripped.getRetryBackoffInitialTime());
         assertEquals(request.getMaxRetries(), tripped.getMaxRetries());
+        assertEquals(request.getRequestsPerSecond(), tripped.getRequestsPerSecond(), 0d);
     }
 
     public void testBulkByTaskStatus() throws IOException {
@@ -102,7 +105,7 @@ public class RoundTripTests extends ESTestCase {
 
     public void testReindexResponse() throws IOException {
         ReindexResponse response = new ReindexResponse(timeValueMillis(randomPositiveLong()), randomStatus(), randomIndexingFailures(),
-                randomSearchFailures());
+                randomSearchFailures(), randomBoolean());
         ReindexResponse tripped = new ReindexResponse();
         roundTrip(response, tripped);
         assertResponseEquals(response, tripped);
@@ -110,7 +113,7 @@ public class RoundTripTests extends ESTestCase {
 
     public void testBulkIndexByScrollResponse() throws IOException {
         BulkIndexByScrollResponse response = new BulkIndexByScrollResponse(timeValueMillis(randomPositiveLong()), randomStatus(),
-                randomIndexingFailures(), randomSearchFailures());
+                randomIndexingFailures(), randomSearchFailures(), randomBoolean());
         BulkIndexByScrollResponse tripped = new BulkIndexByScrollResponse();
         roundTrip(response, tripped);
         assertResponseEquals(response, tripped);
@@ -119,7 +122,7 @@ public class RoundTripTests extends ESTestCase {
     private BulkByScrollTask.Status randomStatus() {
         return new BulkByScrollTask.Status(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
                 randomPositiveInt(), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                random().nextBoolean() ? null : randomSimpleString(random()));
+                parseTimeValue(randomPositiveTimeValue(), "test"), random().nextBoolean() ? null : randomSimpleString(random()));
     }
 
     private List<Failure> randomIndexingFailures() {
@@ -194,5 +197,7 @@ public class RoundTripTests extends ESTestCase {
         assertEquals(expected.getVersionConflicts(), actual.getVersionConflicts());
         assertEquals(expected.getNoops(), actual.getNoops());
         assertEquals(expected.getRetries(), actual.getRetries());
+        assertEquals(expected.getThrottled(), actual.getThrottled());
+        assertEquals(expected.getReasonCancelled(), actual.getReasonCancelled());
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java
index e49afafca46..096967149fb 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java
@@ -19,12 +19,9 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.search.sort.SortOrder;
 
-import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 
 public class UpdateByQueryBasicTests extends UpdateByQueryTestCase {
@@ -64,44 +61,4 @@ public class UpdateByQueryBasicTests extends UpdateByQueryTestCase {
         assertEquals(3, client().prepareGet("test", "test", "3").get().getVersion());
         assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion());
     }
-
-    public void testRefreshIsFalseByDefault() throws Exception {
-        refreshTestCase(null, false);
-    }
-
-    public void testRefreshFalseDoesntMakeVisible() throws Exception {
-        refreshTestCase(false, false);
-    }
-
-    public void testRefreshTrueMakesVisible() throws Exception {
-        refreshTestCase(true, true);
-    }
-
-    /**
-     * Executes an update_by_query on an index with -1 refresh_interval and
-     * checks that the documents are visible properly.
-     */
-    private void refreshTestCase(Boolean refresh, boolean visible) throws Exception {
-        CreateIndexRequestBuilder create = client().admin().indices().prepareCreate("test").setSettings("refresh_interval", -1);
-        create.addMapping("test", "{\"dynamic\": \"false\"}");
-        assertAcked(create);
-        ensureYellow();
-        indexRandom(true, client().prepareIndex("test", "test", "1").setSource("foo", "a"),
-                client().prepareIndex("test", "test", "2").setSource("foo", "a"),
-                client().prepareIndex("test", "test", "3").setSource("foo", "b"),
-                client().prepareIndex("test", "test", "4").setSource("foo", "c"));
-        assertHitCount(client().prepareSearch("test").setQuery(matchQuery("foo", "a")).setSize(0).get(), 0);
-
-        // Now make foo searchable
-        assertAcked(client().admin().indices().preparePutMapping("test").setType("test")
-                .setSource("{\"test\": {\"properties\":{\"foo\": {\"type\": \"text\"}}}}"));
-        UpdateByQueryRequestBuilder update = request().source("test");
-        if (refresh != null) {
-            update.refresh(refresh);
-        }
-        assertThat(update.get(), responseMatcher().updated(4));
-
-        assertHitCount(client().prepareSearch("test").setQuery(matchQuery("foo", "a")).setSize(0).get(), visible ? 2 : 0);
-    }
-
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java
new file mode 100644
index 00000000000..f6780729143
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import java.util.List;
+
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.apache.lucene.util.TestUtil.randomSimpleString;
+import static org.hamcrest.Matchers.arrayWithSize;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.sameInstance;
+
+public class UpdateByQueryRequestTests extends ESTestCase {
+    public void testUpdateByQueryRequestImplementsCompositeIndicesRequestWithDummies() {
+        int numIndices = between(1, 100);
+        String[] indices = new String[numIndices];
+        for (int i = 0; i < numIndices; i++) {
+            indices[i] = randomSimpleString(random(), 1, 30);
+        }
+        UpdateByQueryRequest request = new UpdateByQueryRequest(new SearchRequest(indices));
+        List<? extends IndicesRequest> subRequests = request.subRequests();
+        assertThat(subRequests, hasSize(numIndices + 1));
+        for (int i = 0; i < numIndices; i++) {
+            assertThat(subRequests.get(i).indices(), arrayWithSize(1));
+            assertEquals(indices[i], subRequests.get(i).indices()[0]);
+        }
+        assertThat(subRequests.get(numIndices), sameInstance(request.getSearchRequest()));
+    }
+}
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
index c23e5da95a1..413c8d1c143 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
@@ -21,6 +21,7 @@
   - match: {version_conflicts: 0}
   - match: {batches: 1}
   - match: {failures: []}
+  - match: {throttled_millis: 0}
   - is_true: took
   - is_false: task
 
@@ -53,14 +54,12 @@
   - match: {version_conflicts: 0}
   - match: {batches: 1}
   - match: {failures: []}
+  - match: {throttled_millis: 0}
   - is_true: took
   - is_false: task
 
 ---
 "wait_for_completion=false":
-  - skip:
-      version: "0.0.0 - "
-      reason:  breaks other tests by leaving a running reindex behind
   - do:
       index:
         index:   source
@@ -78,15 +77,23 @@
             index: source
           dest:
             index: dest
+  - is_false: timed_out
   - match: {task: '/.+:\d+/'}
+  - set: {task: task}
   - is_false: updated
   - is_false: version_conflicts
   - is_false: batches
   - is_false: failures
   - is_false: noops
   - is_false: took
+  - is_false: throttled_millis
   - is_false: created
 
+  - do:
+      tasks.list:
+        wait_for_completion: true
+        task_id: $task
+
 ---
 "Response format for version conflict":
   - do:
@@ -122,7 +129,8 @@
   - match: {failures.0.id:     "1"}
   - match: {failures.0.status: 409}
   - match: {failures.0.cause.type:   version_conflict_engine_exception}
-  - match: {failures.0.cause.reason: "[foo][1]: version conflict, document already exists (current version [1])"}
+  # Use a regex so we don't mind if the version isn't always 1. Sometimes it comes out 2.
+  - match: {failures.0.cause.reason: "/\\[foo\\]\\[1\\]:.version.conflict,.document.already.exists.\\(current.version.\\[\\d+\\]\\)/"}
   - match: {failures.0.cause.shard:  /\d+/}
   - match: {failures.0.cause.index:  dest}
   - is_true: took
@@ -158,6 +166,7 @@
   - match: {version_conflicts: 1}
   - match: {batches: 1}
   - match: {failures: []}
+  - match: {throttled_millis: 0}
   - is_true: took
 
 ---
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_throttle.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_throttle.yaml
new file mode 100644
index 00000000000..2543670d5e6
--- /dev/null
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_throttle.yaml
@@ -0,0 +1,53 @@
+---
+"Throttle the request":
+  # Throttling happens between each scroll batch so we need to control the size of the batch by using a single shard
+  # and a small batch size on the request
+  - do:
+      indices.create:
+        index: source
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      2
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      3
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      reindex:
+        requests_per_second: 1
+        body:
+          source:
+            index: source
+            size: 1
+          dest:
+            index: dest
+  - match: {created: 3}
+  - match: {updated: 0}
+  - match: {version_conflicts: 0}
+  - match: {batches: 3}
+  - match: {failures: []}
+  - gt: {throttled_millis: 1000}
+  - lt: {throttled_millis: 4000}
+  - is_true: took
+  - is_false: task
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update-by-query/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update-by-query/10_basic.yaml
index 383e945bbf2..bf54ac5584f 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/update-by-query/10_basic.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update-by-query/10_basic.yaml
@@ -12,11 +12,13 @@
   - do:
       update-by-query:
         index: test
+  - is_false: timed_out
   - match: {updated: 1}
   - match: {version_conflicts: 0}
   - match: {batches: 1}
   - match: {failures: []}
   - match: {noops: 0}
+  - match: {throttled_millis: 0}
   - is_true: took
   - is_false: created # Update by query can't create
   - is_false: task
@@ -37,14 +39,21 @@
         wait_for_completion: false
         index: test
   - match: {task: '/.+:\d+/'}
+  - set: {task: task}
   - is_false: updated
   - is_false: version_conflicts
   - is_false: batches
   - is_false: failures
   - is_false: noops
   - is_false: took
+  - is_false: throttled_millis
   - is_false: created
 
+  - do:
+      tasks.list:
+        wait_for_completion: true
+        task_id: $task
+
 ---
 "Response for version conflict":
   - do:
@@ -80,7 +89,8 @@
   - match: {failures.0.id:     "1"}
   - match: {failures.0.status: 409}
   - match: {failures.0.cause.type:   version_conflict_engine_exception}
-  - match: {failures.0.cause.reason: "[foo][1]: version conflict, current version [2] is different than the one provided [1]"}
+  # Use a regex so we don't mind if the current version isn't always 1. Sometimes it comes out 2.
+  - match: {failures.0.cause.reason: "/\\[foo\\]\\[1\\]:.version.conflict,.current.version.\\[\\d+\\].is.different.than.the.one.provided.\\[\\d+\\]/"}
   - match: {failures.0.cause.shard:  /\d+/}
   - match: {failures.0.cause.index:  test}
   - is_true: took
@@ -117,6 +127,7 @@
   - match: {batches: 1}
   - match: {noops: 0}
   - match: {failures: []}
+  - match: {throttled_millis: 0}
   - is_true: took
 
 ---
@@ -174,6 +185,7 @@
   - match: {version_conflicts: 0}
   - match: {batches: 1}
   - match: {failures: []}
+  - match: {throttled_millis: 0}
   - is_true: took
 
 ---
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update-by-query/70_throttle.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update-by-query/70_throttle.yaml
new file mode 100644
index 00000000000..f0e75b8b2d5
--- /dev/null
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update-by-query/70_throttle.yaml
@@ -0,0 +1,39 @@
+"Throttle the request":
+  # Throttling happens between each scroll batch so we need to control the size of the batch by using a single shard
+  # and a small batch size on the request
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      update-by-query:
+        index: test
+        scroll_size: 1
+        requests_per_second: 1
+  - match: {batches: 3}
+  - match: {updated: 3}
+  - gt: {throttled_millis: 1000}
+  - lt: {throttled_millis: 4000}
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.0-snapshot-bea235f.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index 538d2ad8216..00000000000
--- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-38fda9b86e4f68eb6c9d31fb636a2540da219927
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.0-snapshot-f0aa4fc.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..f2e307d5d98
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+1378905632ff45a9887b267c4b30f7adef415ca4
\ No newline at end of file
diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTransformTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTransformTokenFilterFactory.java
index 6ecdf3888e9..f145ad4ae30 100644
--- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTransformTokenFilterFactory.java
+++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTransformTokenFilterFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.index.analysis;
 import com.ibm.icu.text.Transliterator;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.icu.ICUTransformFilter;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
@@ -36,7 +35,6 @@ public class IcuTransformTokenFilterFactory extends AbstractTokenFilterFactory {
     private final int dir;
     private final Transliterator transliterator;
 
-    @Inject
     public IcuTransformTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
         this.id = settings.get("id", "Null");
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisTestUtils.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisTestUtils.java
deleted file mode 100644
index a952c8982cc..00000000000
--- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisTestUtils.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.analysis;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.ModulesBuilder;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.EnvironmentModule;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.indices.analysis.AnalysisModule;
-import org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin;
-import org.elasticsearch.test.IndexSettingsModule;
-
-import java.io.IOException;
-
-import static org.elasticsearch.common.settings.Settings.settingsBuilder;
-
-public class AnalysisTestUtils {
-
-    public static AnalysisService createAnalysisService(Settings settings) throws IOException {
-        Index index = new Index("test", "_na_");
-        Settings indexSettings = settingsBuilder().put(settings)
-                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                .build();
-        AnalysisModule analysisModule = new AnalysisModule(new Environment(settings));
-        new AnalysisICUPlugin().onModule(analysisModule);
-        Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings),
-                new EnvironmentModule(new Environment(settings)), analysisModule)
-                .createInjector();
-        final AnalysisService analysisService = parentInjector.getInstance(AnalysisRegistry.class).build(IndexSettingsModule.newIndexSettings(index, indexSettings));
-        return analysisService;
-    }
-}
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java
index efd60427e23..b399dfd34f4 100644
--- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java
@@ -20,21 +20,19 @@
 package org.elasticsearch.index.analysis;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 
-import static org.elasticsearch.common.settings.Settings.settingsBuilder;
-import static org.elasticsearch.index.analysis.AnalysisTestUtils.createAnalysisService;
 import static org.hamcrest.Matchers.instanceOf;
 /**
  */
 public class SimpleIcuAnalysisTests extends ESTestCase {
     public void testDefaultsIcuAnalysis() throws IOException {
-        Settings settings = settingsBuilder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
-        AnalysisService analysisService = createAnalysisService(settings);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"),
+            Settings.EMPTY, new AnalysisICUPlugin()::onModule);
 
         TokenizerFactory tokenizerFactory = analysisService.tokenizer("icu_tokenizer");
         assertThat(tokenizerFactory, instanceOf(IcuTokenizerFactory.class));
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java
index 632f3f539d6..adf1faaf92f 100644
--- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java
@@ -27,13 +27,13 @@ import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.KeywordTokenizer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.io.StringReader;
 
-import static org.elasticsearch.index.analysis.AnalysisTestUtils.createAnalysisService;
 import static org.hamcrest.Matchers.equalTo;
 
 // Tests borrowed from Solr's Icu collation key filter factory test.
@@ -46,12 +46,11 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
     */
     public void testBasicUsage() throws Exception {
         Settings settings = Settings.settingsBuilder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .put("index.analysis.filter.myCollator.type", "icu_collation")
                 .put("index.analysis.filter.myCollator.language", "tr")
                 .put("index.analysis.filter.myCollator.strength", "primary")
                 .build();
-        AnalysisService analysisService = createAnalysisService(settings);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollatesToSame(filterFactory, "I WİLL USE TURKİSH CASING", "ı will use turkish casıng");
@@ -62,13 +61,12 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
     */
     public void testNormalization() throws IOException {
         Settings settings = Settings.settingsBuilder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .put("index.analysis.filter.myCollator.type", "icu_collation")
                 .put("index.analysis.filter.myCollator.language", "tr")
                 .put("index.analysis.filter.myCollator.strength", "primary")
                 .put("index.analysis.filter.myCollator.decomposition", "canonical")
                 .build();
-        AnalysisService analysisService = createAnalysisService(settings);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollatesToSame(filterFactory, "I W\u0049\u0307LL USE TURKİSH CASING", "ı will use turkish casıng");
@@ -79,13 +77,12 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
     */
     public void testSecondaryStrength() throws IOException {
         Settings settings = Settings.settingsBuilder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .put("index.analysis.filter.myCollator.type", "icu_collation")
                 .put("index.analysis.filter.myCollator.language", "en")
                 .put("index.analysis.filter.myCollator.strength", "secondary")
                 .put("index.analysis.filter.myCollator.decomposition", "no")
                 .build();
-        AnalysisService analysisService = createAnalysisService(settings);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollatesToSame(filterFactory, "TESTING", "testing");
@@ -97,13 +94,12 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
     */
     public void testIgnorePunctuation() throws IOException {
         Settings settings = Settings.settingsBuilder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .put("index.analysis.filter.myCollator.type", "icu_collation")
                 .put("index.analysis.filter.myCollator.language", "en")
                 .put("index.analysis.filter.myCollator.strength", "primary")
                 .put("index.analysis.filter.myCollator.alternate", "shifted")
                 .build();
-        AnalysisService analysisService = createAnalysisService(settings);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollatesToSame(filterFactory, "foo-bar", "foo bar");
@@ -115,14 +111,13 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
     */
     public void testIgnoreWhitespace() throws IOException {
         Settings settings = Settings.settingsBuilder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .put("index.analysis.filter.myCollator.type", "icu_collation")
                 .put("index.analysis.filter.myCollator.language", "en")
                 .put("index.analysis.filter.myCollator.strength", "primary")
                 .put("index.analysis.filter.myCollator.alternate", "shifted")
                 .put("index.analysis.filter.myCollator.variableTop", " ")
                 .build();
-        AnalysisService analysisService = createAnalysisService(settings);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollatesToSame(filterFactory, "foo bar", "foobar");
@@ -136,12 +131,11 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
     */
     public void testNumerics() throws IOException {
         Settings settings = Settings.settingsBuilder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .put("index.analysis.filter.myCollator.type", "icu_collation")
                 .put("index.analysis.filter.myCollator.language", "en")
                 .put("index.analysis.filter.myCollator.numeric", "true")
                 .build();
-        AnalysisService analysisService = createAnalysisService(settings);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollation(filterFactory, "foobar-9", "foobar-10", -1);
@@ -153,13 +147,12 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
     */
     public void testIgnoreAccentsButNotCase() throws IOException {
         Settings settings = Settings.settingsBuilder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .put("index.analysis.filter.myCollator.type", "icu_collation")
                 .put("index.analysis.filter.myCollator.language", "en")
                 .put("index.analysis.filter.myCollator.strength", "primary")
                 .put("index.analysis.filter.myCollator.caseLevel", "true")
                 .build();
-        AnalysisService analysisService = createAnalysisService(settings);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollatesToSame(filterFactory, "résumé", "resume");
@@ -174,13 +167,12 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
     */
     public void testUpperCaseFirst() throws IOException {
         Settings settings = Settings.settingsBuilder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .put("index.analysis.filter.myCollator.type", "icu_collation")
                 .put("index.analysis.filter.myCollator.language", "en")
                 .put("index.analysis.filter.myCollator.strength", "tertiary")
                 .put("index.analysis.filter.myCollator.caseFirst", "upper")
                 .build();
-        AnalysisService analysisService = createAnalysisService(settings);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollation(filterFactory, "Resume", "resume", -1);
@@ -204,12 +196,11 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
         String tailoredRules = tailoredCollator.getRules();
 
         Settings settings = Settings.settingsBuilder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .put("index.analysis.filter.myCollator.type", "icu_collation")
                 .put("index.analysis.filter.myCollator.rules", tailoredRules)
                 .put("index.analysis.filter.myCollator.strength", "primary")
                 .build();
-        AnalysisService analysisService = createAnalysisService(settings);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollatesToSame(filterFactory, "Töne", "Toene");
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java
index 7ebb783d1db..749b04b2260 100644
--- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java
@@ -22,12 +22,12 @@ package org.elasticsearch.index.analysis;
 import com.ibm.icu.text.Normalizer2;
 import org.apache.lucene.analysis.CharFilter;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.StringReader;
 
-import static org.elasticsearch.index.analysis.AnalysisTestUtils.createAnalysisService;
 
 /**
  * Test
@@ -35,10 +35,9 @@ import static org.elasticsearch.index.analysis.AnalysisTestUtils.createAnalysisS
 public class SimpleIcuNormalizerCharFilterTests extends ESTestCase {
     public void testDefaultSetting() throws Exception {
         Settings settings = Settings.settingsBuilder()
-            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
             .put("index.analysis.char_filter.myNormalizerChar.type", "icu_normalizer")
             .build();
-        AnalysisService analysisService = createAnalysisService(settings);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
         CharFilterFactory charFilterFactory = analysisService.charFilter("myNormalizerChar");
 
         String input = "ʰ㌰゙5℃№㈱㌘，バッファーの正規化のテスト．㋐㋑㋒㋓㋔ｶｷｸｹｺｻﾞｼﾞｽﾞｾﾞｿﾞg̈각/각நிเกषिchkʷक्षि";
@@ -58,12 +57,11 @@ public class SimpleIcuNormalizerCharFilterTests extends ESTestCase {
 
     public void testNameAndModeSetting() throws Exception {
         Settings settings = Settings.settingsBuilder()
-            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
             .put("index.analysis.char_filter.myNormalizerChar.type", "icu_normalizer")
             .put("index.analysis.char_filter.myNormalizerChar.name", "nfkc")
             .put("index.analysis.char_filter.myNormalizerChar.mode", "decompose")
             .build();
-        AnalysisService analysisService = createAnalysisService(settings);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
         CharFilterFactory charFilterFactory = analysisService.charFilter("myNormalizerChar");
 
         String input = "ʰ㌰゙5℃№㈱㌘，バッファーの正規化のテスト．㋐㋑㋒㋓㋔ｶｷｸｹｺｻﾞｼﾞｽﾞｾﾞｿﾞg̈각/각நிเกषिchkʷक्षि";
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.0-snapshot-bea235f.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index b90115da4ab..00000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-352fea7a169ada6a7ae18e4ec34559496e09b465
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.0-snapshot-f0aa4fc.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..7bf3eb5333d
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+49acd38e206d9c2fe28269fcba9b752d3b605e0e
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiAnalyzerProvider.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiAnalyzerProvider.java
index 8aa8ff3c1dd..21d9b804055 100644
--- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiAnalyzerProvider.java
+++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiAnalyzerProvider.java
@@ -23,7 +23,6 @@ import org.apache.lucene.analysis.ja.JapaneseAnalyzer;
 import org.apache.lucene.analysis.ja.JapaneseTokenizer;
 import org.apache.lucene.analysis.ja.dict.UserDictionary;
 import org.apache.lucene.analysis.util.CharArraySet;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
@@ -36,7 +35,6 @@ public class KuromojiAnalyzerProvider extends AbstractIndexAnalyzerProvider<Japa
 
     private final JapaneseAnalyzer analyzer;
 
-    @Inject
     public KuromojiAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
         final Set<?> stopWords = Analysis.parseStopWords(env, settings, JapaneseAnalyzer.getDefaultStopSet());
diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiBaseFormFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiBaseFormFilterFactory.java
index e191d78198f..aa035d9edfd 100644
--- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiBaseFormFilterFactory.java
+++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiBaseFormFilterFactory.java
@@ -21,14 +21,12 @@ package org.elasticsearch.index.analysis;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.ja.JapaneseBaseFormFilter;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 
 public class KuromojiBaseFormFilterFactory extends AbstractTokenFilterFactory {
 
-    @Inject
     public KuromojiBaseFormFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
     }
diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiKatakanaStemmerFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiKatakanaStemmerFactory.java
index ebebdcb6bba..491f48e34c1 100644
--- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiKatakanaStemmerFactory.java
+++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiKatakanaStemmerFactory.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.analysis;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.ja.JapaneseKatakanaStemFilter;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
@@ -30,7 +29,6 @@ public class KuromojiKatakanaStemmerFactory extends AbstractTokenFilterFactory {
 
     private final int minimumLength;
 
-    @Inject
     public KuromojiKatakanaStemmerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
         minimumLength = settings.getAsInt("minimum_length", JapaneseKatakanaStemFilter.DEFAULT_MINIMUM_LENGTH);
diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiNumberFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiNumberFilterFactory.java
new file mode 100644
index 00000000000..cb6b478957a
--- /dev/null
+++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiNumberFilterFactory.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.ja.JapaneseNumberFilter;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+
+public class KuromojiNumberFilterFactory extends AbstractTokenFilterFactory {
+
+    public KuromojiNumberFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+        super(indexSettings, name, settings);
+    }
+
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        return new JapaneseNumberFilter(tokenStream);
+    }
+}
diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiReadingFormFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiReadingFormFilterFactory.java
index 59d1088fd1b..d0eb0cecdb9 100644
--- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiReadingFormFilterFactory.java
+++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiReadingFormFilterFactory.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.analysis;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.ja.JapaneseReadingFormFilter;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
@@ -30,7 +29,6 @@ public class KuromojiReadingFormFilterFactory extends AbstractTokenFilterFactory
 
     private final boolean useRomaji;
 
-    @Inject
     public KuromojiReadingFormFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
         useRomaji = settings.getAsBoolean("use_romaji", false);
diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiTokenizerFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiTokenizerFactory.java
index 87e08c757b4..9e41621525a 100644
--- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiTokenizerFactory.java
+++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiTokenizerFactory.java
@@ -36,9 +36,13 @@ import java.io.Reader;
 public class KuromojiTokenizerFactory extends AbstractTokenizerFactory {
 
     private static final String USER_DICT_OPTION = "user_dictionary";
+    private static final String NBEST_COST = "nbest_cost";
+    private static final String NBEST_EXAMPLES = "nbest_examples";
 
     private final UserDictionary userDictionary;
     private final Mode mode;
+    private final String nBestExamples;
+    private final int nBestCost;
 
     private boolean discartPunctuation;
 
@@ -47,6 +51,8 @@ public class KuromojiTokenizerFactory extends AbstractTokenizerFactory {
         mode = getMode(settings);
         userDictionary = getUserDictionary(env, settings);
         discartPunctuation = settings.getAsBoolean("discard_punctuation", true);
+        nBestCost = settings.getAsInt(NBEST_COST, -1);
+        nBestExamples = settings.get(NBEST_EXAMPLES);
     }
 
     public static UserDictionary getUserDictionary(Environment env, Settings settings) {
@@ -83,7 +89,13 @@ public class KuromojiTokenizerFactory extends AbstractTokenizerFactory {
 
     @Override
     public Tokenizer create() {
-        return new JapaneseTokenizer(userDictionary, discartPunctuation, mode);
+        JapaneseTokenizer t = new JapaneseTokenizer(userDictionary, discartPunctuation, mode);
+        int nBestCost = this.nBestCost;
+        if (nBestExamples != null) {
+            nBestCost = Math.max(nBestCost, t.calcNBestCost(nBestExamples));
+        }
+        t.setNBestCost(nBestCost);
+        return t;
     }
 
 }
diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java
index 6c0a15f2e39..4208b1be504 100644
--- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java
+++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java
@@ -24,6 +24,7 @@ import org.elasticsearch.index.analysis.KuromojiAnalyzerProvider;
 import org.elasticsearch.index.analysis.KuromojiBaseFormFilterFactory;
 import org.elasticsearch.index.analysis.KuromojiIterationMarkCharFilterFactory;
 import org.elasticsearch.index.analysis.KuromojiKatakanaStemmerFactory;
+import org.elasticsearch.index.analysis.KuromojiNumberFilterFactory;
 import org.elasticsearch.index.analysis.KuromojiPartOfSpeechFilterFactory;
 import org.elasticsearch.index.analysis.KuromojiReadingFormFilterFactory;
 import org.elasticsearch.index.analysis.KuromojiTokenizerFactory;
@@ -55,5 +56,6 @@ public class AnalysisKuromojiPlugin extends Plugin {
         module.registerTokenFilter("kuromoji_readingform", KuromojiReadingFormFilterFactory::new);
         module.registerTokenFilter("kuromoji_stemmer", KuromojiKatakanaStemmerFactory::new);
         module.registerTokenFilter("ja_stop", JapaneseStopTokenFilterFactory::new);
+        module.registerTokenFilter("kuromoji_number", KuromojiNumberFilterFactory::new);
     }
 }
diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java
index 0b3f026b010..93aeddeee5f 100644
--- a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java
+++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java
@@ -26,18 +26,11 @@ import org.apache.lucene.analysis.ja.JapaneseTokenizer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.plugin.analysis.kuromoji.AnalysisKuromojiPlugin;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.test.InternalSettingsPlugin;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -75,6 +68,9 @@ public class KuromojiAnalysisTests extends ESTestCase {
         filterFactory = analysisService.tokenFilter("ja_stop");
         assertThat(filterFactory, instanceOf(JapaneseStopTokenFilterFactory.class));
 
+        filterFactory = analysisService.tokenFilter("kuromoji_number");
+        assertThat(filterFactory, instanceOf(KuromojiNumberFilterFactory.class));
+
         NamedAnalyzer analyzer = analysisService.analyzer("kuromoji");
         assertThat(analyzer.analyzer(), instanceOf(JapaneseAnalyzer.class));
 
@@ -187,32 +183,22 @@ public class KuromojiAnalysisTests extends ESTestCase {
         assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
     }
 
-    public AnalysisService createAnalysisService() throws IOException {
-        InputStream empty_dict = getClass().getResourceAsStream("empty_user_dict.txt");
-        InputStream dict = getClass().getResourceAsStream("user_dict.txt");
+    private static AnalysisService createAnalysisService() throws IOException {
+        InputStream empty_dict = KuromojiAnalysisTests.class.getResourceAsStream("empty_user_dict.txt");
+        InputStream dict = KuromojiAnalysisTests.class.getResourceAsStream("user_dict.txt");
         Path home = createTempDir();
         Path config = home.resolve("config");
         Files.createDirectory(config);
         Files.copy(empty_dict, config.resolve("empty_user_dict.txt"));
         Files.copy(dict, config.resolve("user_dict.txt"));
-
         String json = "/org/elasticsearch/index/analysis/kuromoji_analysis.json";
+
         Settings settings = Settings.settingsBuilder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), home)
-                .loadFromStream(json, getClass().getResourceAsStream(json))
-                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                .build();
-        final SettingsModule settingsModule = new SettingsModule(settings);
-        settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
-        Index index = new Index("test", "_na_");
-
-        AnalysisModule analysisModule = new AnalysisModule(new Environment(settings));
-        new AnalysisKuromojiPlugin().onModule(analysisModule);
-        Injector parentInjector = new ModulesBuilder().add(settingsModule,
-                new EnvironmentModule(new Environment(settings)), analysisModule)
-                .createInjector();
-
-        return parentInjector.getInstance(AnalysisRegistry.class).build(IndexSettingsModule.newIndexSettings(index, settings));
+            .loadFromStream(json, KuromojiAnalysisTests.class.getResourceAsStream(json))
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .build();
+        Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), home).build();
+        return createAnalysisService(new Index("test", "_na_"), nodeSettings, settings, new AnalysisKuromojiPlugin()::onModule);
     }
 
     public static void assertSimpleTSOutput(TokenStream stream,
@@ -260,4 +246,49 @@ public class KuromojiAnalysisTests extends ESTestCase {
         TokenizerFactory tokenizerFactory = analysisService.tokenizer("kuromoji_empty_user_dict");
         assertThat(tokenizerFactory, instanceOf(KuromojiTokenizerFactory.class));
     }
+
+    public void testNbestCost() throws IOException {
+        AnalysisService analysisService = createAnalysisService();
+        TokenizerFactory tokenizerFactory = analysisService.tokenizer("kuromoji_nbest_cost");
+        String source = "鳩山積み";
+        String[] expected = new String[] {"鳩", "鳩山", "山積み", "積み"};
+
+        Tokenizer tokenizer = tokenizerFactory.create();
+        tokenizer.setReader(new StringReader(source));
+        assertSimpleTSOutput(tokenizer, expected);
+    }
+
+    public void testNbestExample() throws IOException {
+        AnalysisService analysisService = createAnalysisService();
+        TokenizerFactory tokenizerFactory = analysisService.tokenizer("kuromoji_nbest_examples");
+        String source = "鳩山積み";
+        String[] expected = new String[] {"鳩", "鳩山", "山積み", "積み"};
+
+        Tokenizer tokenizer = tokenizerFactory.create();
+        tokenizer.setReader(new StringReader(source));
+        assertSimpleTSOutput(tokenizer, expected);
+    }
+
+    public void testNbestBothOptions() throws IOException {
+        AnalysisService analysisService = createAnalysisService();
+        TokenizerFactory tokenizerFactory = analysisService.tokenizer("kuromoji_nbest_both");
+        String source = "鳩山積み";
+        String[] expected = new String[] {"鳩", "鳩山", "山積み", "積み"};
+
+        Tokenizer tokenizer = tokenizerFactory.create();
+        tokenizer.setReader(new StringReader(source));
+        assertSimpleTSOutput(tokenizer, expected);
+
+    }
+
+    public void testNumberFilterFactory() throws Exception {
+        AnalysisService analysisService = createAnalysisService();
+        TokenFilterFactory tokenFilter = analysisService.tokenFilter("kuromoji_number");
+        assertThat(tokenFilter, instanceOf(KuromojiNumberFilterFactory.class));
+        String source = "本日十万二千五百円のワインを買った";
+        String[] expected = new String[]{"本日", "102500", "円", "の", "ワイン", "を", "買っ", "た"};
+        Tokenizer tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.Mode.SEARCH);
+        tokenizer.setReader(new StringReader(source));
+        assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
+    }
 }
diff --git a/plugins/analysis-kuromoji/src/test/resources/org/elasticsearch/index/analysis/kuromoji_analysis.json b/plugins/analysis-kuromoji/src/test/resources/org/elasticsearch/index/analysis/kuromoji_analysis.json
index 58ed015b850..d0f94a2117b 100644
--- a/plugins/analysis-kuromoji/src/test/resources/org/elasticsearch/index/analysis/kuromoji_analysis.json
+++ b/plugins/analysis-kuromoji/src/test/resources/org/elasticsearch/index/analysis/kuromoji_analysis.json
@@ -18,7 +18,6 @@
                     "type": "ja_stop",
                     "stopwords": ["_japanese_", "スピード"]
                 }
-                
             },
 
             "char_filter":{
@@ -48,6 +47,19 @@
                 "kuromoji_user_dict" : {
                     "type":"kuromoji_tokenizer",
                     "user_dictionary":"user_dict.txt"
+                },
+                "kuromoji_nbest_cost" : {
+                    "type": "kuromoji_tokenizer",
+                    "nbest_cost" : "2000"
+                },
+                "kuromoji_nbest_examples" : {
+                    "type": "kuromoji_tokenizer",
+                    "nbest_examples" : "/鳩山積み-鳩山/鳩山積み-鳩/"
+                },
+                "kuromoji_nbest_both" : {
+                    "type": "kuromoji_tokenizer",
+                    "nbest_examples" : "/鳩山積み-鳩山/鳩山積み-鳩/",
+                    "nbest_cost" : "1000"
                 }
             },
             "analyzer" : {
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.0-snapshot-bea235f.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index 7cbe648e0bd..00000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-445f5ea7822d0dd6b91364ec119cd6cb4635d285
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.0-snapshot-f0aa4fc.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..8f08fe26980
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+7c11723d7d4dc3b1c9bf80089cfc2de7bc8a2b6e
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticTokenFilterFactory.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticTokenFilterFactory.java
index e33f1f1e7e2..75da19c0a3c 100644
--- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticTokenFilterFactory.java
+++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticTokenFilterFactory.java
@@ -38,7 +38,6 @@ import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.phonetic.BeiderMorseFilter;
 import org.apache.lucene.analysis.phonetic.DoubleMetaphoneFilter;
 import org.apache.lucene.analysis.phonetic.PhoneticFilter;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
@@ -58,7 +57,6 @@ public class PhoneticTokenFilterFactory extends AbstractTokenFilterFactory {
     private NameType nametype;
     private RuleType ruletype;
 
-    @Inject
     public PhoneticTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
         this.languageset = null;
diff --git a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java
index f3d1d12f45a..688394b6844 100644
--- a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java
+++ b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java
@@ -21,18 +21,10 @@ package org.elasticsearch.index.analysis;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.plugin.analysis.AnalysisPhoneticPlugin;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.test.InternalSettingsPlugin;
 import org.hamcrest.MatcherAssert;
 
 import java.io.IOException;
@@ -47,22 +39,10 @@ public class SimplePhoneticAnalysisTests extends ESTestCase {
         String yaml = "/org/elasticsearch/index/analysis/phonetic-1.yml";
         Settings settings = settingsBuilder().loadFromStream(yaml, getClass().getResourceAsStream(yaml))
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .build();
-        AnalysisService analysisService = testSimpleConfiguration(settings);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings,
+            new AnalysisPhoneticPlugin()::onModule);
         TokenFilterFactory filterFactory = analysisService.tokenFilter("phonetic");
         MatcherAssert.assertThat(filterFactory, instanceOf(PhoneticTokenFilterFactory.class));
     }
-
-    private AnalysisService testSimpleConfiguration(Settings settings) throws IOException {
-        Index index = new Index("test", "_na_");
-        AnalysisModule analysisModule = new AnalysisModule(new Environment(settings));
-        new AnalysisPhoneticPlugin().onModule(analysisModule);
-        SettingsModule settingsModule = new SettingsModule(settings);
-        settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
-        Injector parentInjector = new ModulesBuilder().add(settingsModule,
-                new EnvironmentModule(new Environment(settings)), analysisModule)
-                .createInjector();
-        return parentInjector.getInstance(AnalysisRegistry.class).build(IndexSettingsModule.newIndexSettings(index, settings));
-    }
 }
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.0-snapshot-bea235f.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index 03c96786de2..00000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0b216b7b9ff583bc1382edc8adfee4d4acd02859
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.0-snapshot-f0aa4fc.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..bf5e5da8dcf
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+654d961bd4975a3cb13388d86d72fefb6994f659
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseAnalyzerProvider.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseAnalyzerProvider.java
index 22fcf238725..591912b8fa3 100644
--- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseAnalyzerProvider.java
+++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseAnalyzerProvider.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.analysis;
 
 import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
@@ -31,7 +30,6 @@ public class SmartChineseAnalyzerProvider extends AbstractIndexAnalyzerProvider<
 
     private final SmartChineseAnalyzer analyzer;
 
-    @Inject
     public SmartChineseAnalyzerProvider(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
 
diff --git a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java
index 76761a67c9f..0fcc42643d4 100644
--- a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java
+++ b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java
@@ -19,44 +19,21 @@
 
 package org.elasticsearch.index.analysis;
 
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.plugin.analysis.smartcn.AnalysisSmartChinesePlugin;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.test.InternalSettingsPlugin;
 import org.hamcrest.MatcherAssert;
 
 import java.io.IOException;
 
-import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.hamcrest.Matchers.instanceOf;
 
 /**
  */
 public class SimpleSmartChineseAnalysisTests extends ESTestCase {
     public void testDefaultsIcuAnalysis() throws IOException {
-        Index index = new Index("test", "_na_");
-        Settings settings = settingsBuilder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                .build();
-        AnalysisModule analysisModule = new AnalysisModule(new Environment(settings));
-        new AnalysisSmartChinesePlugin().onModule(analysisModule);
-        SettingsModule settingsModule = new SettingsModule(settings);
-        settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
-        Injector parentInjector = new ModulesBuilder().add(settingsModule,
-                new EnvironmentModule(new Environment(settings)), analysisModule)
-                .createInjector();
-        final AnalysisService analysisService = parentInjector.getInstance(AnalysisRegistry.class).build(IndexSettingsModule.newIndexSettings(index, settings));
+        final AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, new AnalysisSmartChinesePlugin()::onModule);
         TokenizerFactory tokenizerFactory = analysisService.tokenizer("smartcn_tokenizer");
         MatcherAssert.assertThat(tokenizerFactory, instanceOf(SmartChineseTokenizerTokenizerFactory.class));
     }
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.0-snapshot-bea235f.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.0-snapshot-bea235f.jar.sha1
deleted file mode 100644
index f27a98f63ba..00000000000
--- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.0-snapshot-bea235f.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8d161a8c7e5b5b82f64dc5df2ca46197a3716672
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.0-snapshot-f0aa4fc.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.0-snapshot-f0aa4fc.jar.sha1
new file mode 100644
index 00000000000..ed0dc51b97c
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.0-snapshot-f0aa4fc.jar.sha1
@@ -0,0 +1 @@
+0f408ac498782617a0f80d6a295d82f6d3609499
\ No newline at end of file
diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java
index 8f76c908e4b..9bfcc2c2f3f 100644
--- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java
+++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java
@@ -21,46 +21,22 @@ package org.elasticsearch.index.analysis;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.pl.PolishAnalyzer;
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.EnvironmentModule;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.analysis.pl.PolishStemTokenFilterFactory;
-import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.plugin.analysis.stempel.AnalysisStempelPlugin;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.test.InternalSettingsPlugin;
 import org.hamcrest.MatcherAssert;
 
 import java.io.IOException;
 
-import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.hamcrest.Matchers.instanceOf;
 
 /**
  */
 public class PolishAnalysisTests extends ESTestCase {
     public void testDefaultsPolishAnalysis() throws IOException {
-        Settings settings = settingsBuilder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                .build();
-
-
-        AnalysisModule analysisModule = new AnalysisModule(new Environment(settings));
-        new AnalysisStempelPlugin().onModule(analysisModule);
-        SettingsModule settingsModule = new SettingsModule(settings);
-        settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
-        Injector parentInjector = new ModulesBuilder().add(settingsModule,
-                new EnvironmentModule(new Environment(settings)), analysisModule)
-                .createInjector();
-
-        final AnalysisService analysisService = parentInjector.getInstance(AnalysisRegistry.class).build(IndexSettingsModule.newIndexSettings("test", settings));
+        final AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, new AnalysisStempelPlugin()::onModule);
         TokenFilterFactory tokenizerFactory = analysisService.tokenFilter("polish_stem");
         MatcherAssert.assertThat(tokenizerFactory, instanceOf(PolishStemTokenFilterFactory.class));
 
diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java
index 890f4eceec1..193cfea6811 100644
--- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java
+++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java
@@ -24,20 +24,10 @@ import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.KeywordTokenizer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.plugin.analysis.stempel.AnalysisStempelPlugin;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.test.InternalSettingsPlugin;
 
 import java.io.IOException;
 import java.io.StringReader;
@@ -57,11 +47,9 @@ public class SimplePolishTokenFilterTests extends ESTestCase {
     private void testToken(String source, String expected) throws IOException {
         Index index = new Index("test", "_na_");
         Settings settings = Settings.settingsBuilder()
-                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .put("index.analysis.filter.myStemmer.type", "polish_stem")
                 .build();
-        AnalysisService analysisService = createAnalysisService(index, settings);
+        AnalysisService analysisService = createAnalysisService(index, settings, new AnalysisStempelPlugin()::onModule);
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myStemmer");
 
@@ -77,12 +65,8 @@ public class SimplePolishTokenFilterTests extends ESTestCase {
     }
 
     private void testAnalyzer(String source, String... expected_terms) throws IOException {
-        Index index = new Index("test", "_na_");
-        Settings settings = Settings.settingsBuilder()
-                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                .build();
-        AnalysisService analysisService = createAnalysisService(index, settings);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY,
+            new AnalysisStempelPlugin()::onModule);
 
         Analyzer analyzer = analysisService.analyzer("polish").analyzer();
 
@@ -97,14 +81,4 @@ public class SimplePolishTokenFilterTests extends ESTestCase {
         }
     }
 
-    private AnalysisService createAnalysisService(Index index, Settings settings) throws IOException {
-        AnalysisModule analysisModule = new AnalysisModule(new Environment(settings));
-        new AnalysisStempelPlugin().onModule(analysisModule);
-        SettingsModule settingsModule = new SettingsModule(settings);
-        settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
-        Injector parentInjector = new ModulesBuilder().add(settingsModule,
-                new EnvironmentModule(new Environment(settings)), analysisModule)
-                .createInjector();
-        return parentInjector.getInstance(AnalysisRegistry.class).build(IndexSettingsModule.newIndexSettings(index, settings));
-    }
 }
diff --git a/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryTests.java b/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryTests.java
index 232b056535c..3c595b1ab16 100644
--- a/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryTests.java
+++ b/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryTests.java
@@ -276,7 +276,7 @@ public class DeleteByQueryTests extends ESIntegTestCase {
         assertSearchContextsClosed();
     }
 
-    public void testConcurrentDeleteByQueriesOnDifferentDocs() throws Exception {
+    public void testConcurrentDeleteByQueriesOnDifferentDocs() throws Throwable {
         createIndex("test");
         ensureGreen();
 
@@ -324,18 +324,17 @@ public class DeleteByQueryTests extends ESIntegTestCase {
 
         Throwable assertionError = exceptionHolder.get();
         if (assertionError != null) {
-            assertionError.printStackTrace();
+            throw assertionError;
         }
-        assertThat(assertionError + " should be null", assertionError, nullValue());
-        refresh();
 
+        refresh();
         for (int i = 0; i < threads.length; i++) {
             assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.termQuery("field", i)).get(), 0);
         }
         assertSearchContextsClosed();
     }
 
-    public void testConcurrentDeleteByQueriesOnSameDocs() throws Exception {
+    public void testConcurrentDeleteByQueriesOnSameDocs() throws Throwable {
         assertAcked(prepareCreate("test").setSettings(Settings.settingsBuilder().put("index.refresh_interval", -1)));
         ensureGreen();
 
@@ -386,9 +385,8 @@ public class DeleteByQueryTests extends ESIntegTestCase {
 
         Throwable assertionError = exceptionHolder.get();
         if (assertionError != null) {
-            assertionError.printStackTrace();
+            throw assertionError;
         }
-        assertThat(assertionError + " should be null", assertionError, nullValue());
         assertHitCount(client().prepareSearch("test").setSize(0).get(), 0L);
         assertThat(deleted.get(), equalTo(docs));
         assertSearchContextsClosed();
@@ -445,4 +443,4 @@ public class DeleteByQueryTests extends ESIntegTestCase {
             }
         });
     }
-}
\ No newline at end of file
+}
diff --git a/plugins/discovery-azure/build.gradle b/plugins/discovery-azure/build.gradle
index ec4ef7cb625..1dd2aa26f23 100644
--- a/plugins/discovery-azure/build.gradle
+++ b/plugins/discovery-azure/build.gradle
@@ -23,7 +23,7 @@ esplugin {
 }
 
 versions << [
-  'azure': '0.9.0',
+  'azure': '0.9.3',
   'jersey': '1.13'
 ]
 
diff --git a/plugins/discovery-azure/licenses/azure-core-0.9.0.jar.sha1 b/plugins/discovery-azure/licenses/azure-core-0.9.0.jar.sha1
deleted file mode 100644
index f9696307afe..00000000000
--- a/plugins/discovery-azure/licenses/azure-core-0.9.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-050719f91deceed1be1aaf87e85099a861295fa2
\ No newline at end of file
diff --git a/plugins/discovery-azure/licenses/azure-core-0.9.3.jar.sha1 b/plugins/discovery-azure/licenses/azure-core-0.9.3.jar.sha1
new file mode 100644
index 00000000000..5947972663e
--- /dev/null
+++ b/plugins/discovery-azure/licenses/azure-core-0.9.3.jar.sha1
@@ -0,0 +1 @@
+7fe32241b738aad0f700f4277fa998230c144ae7
\ No newline at end of file
diff --git a/plugins/discovery-azure/licenses/azure-svc-mgmt-compute-0.9.0.jar.sha1 b/plugins/discovery-azure/licenses/azure-svc-mgmt-compute-0.9.0.jar.sha1
deleted file mode 100644
index c971d7c5724..00000000000
--- a/plugins/discovery-azure/licenses/azure-svc-mgmt-compute-0.9.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-887ca8ee5564e8ba2351e6b5db2a1293a8d04674
\ No newline at end of file
diff --git a/plugins/discovery-azure/licenses/azure-svc-mgmt-compute-0.9.3.jar.sha1 b/plugins/discovery-azure/licenses/azure-svc-mgmt-compute-0.9.3.jar.sha1
new file mode 100644
index 00000000000..d427170d578
--- /dev/null
+++ b/plugins/discovery-azure/licenses/azure-svc-mgmt-compute-0.9.3.jar.sha1
@@ -0,0 +1 @@
+602d3e6f5a9f058c2439e8fdf1270cddc811b440
\ No newline at end of file
diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeService.java b/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeService.java
index 0c665c138b8..acc1e76bde4 100644
--- a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeService.java
+++ b/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeService.java
@@ -22,26 +22,36 @@ package org.elasticsearch.cloud.azure.management;
 import com.microsoft.windowsazure.core.utils.KeyStoreType;
 import com.microsoft.windowsazure.management.compute.models.HostedServiceGetDetailedResponse;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.azure.AzureUnicastHostsProvider;
 
 public interface AzureComputeService {
 
     final class Management {
-        public static final Setting<String> SUBSCRIPTION_ID_SETTING = Setting.simpleString("cloud.azure.management.subscription.id", false, Setting.Scope.CLUSTER);
-        public static final Setting<String> SERVICE_NAME_SETTING = Setting.simpleString("cloud.azure.management.cloud.service.name", false, Setting.Scope.CLUSTER);
+        public static final Setting<String> SUBSCRIPTION_ID_SETTING =
+            Setting.simpleString("cloud.azure.management.subscription.id", Property.NodeScope, Property.Filtered);
+        public static final Setting<String> SERVICE_NAME_SETTING =
+            Setting.simpleString("cloud.azure.management.cloud.service.name", Property.NodeScope);
 
         // Keystore settings
-        public static final Setting<String> KEYSTORE_PATH_SETTING = Setting.simpleString("cloud.azure.management.keystore.path", false, Setting.Scope.CLUSTER);
-        public static final Setting<String> KEYSTORE_PASSWORD_SETTING = Setting.simpleString("cloud.azure.management.keystore.password", false, Setting.Scope.CLUSTER);
-        public static final Setting<KeyStoreType> KEYSTORE_TYPE_SETTING = new Setting<>("cloud.azure.management.keystore.type", KeyStoreType.pkcs12.name(), KeyStoreType::fromString, false, Setting.Scope.CLUSTER);
+        public static final Setting<String> KEYSTORE_PATH_SETTING =
+            Setting.simpleString("cloud.azure.management.keystore.path", Property.NodeScope, Property.Filtered);
+        public static final Setting<String> KEYSTORE_PASSWORD_SETTING =
+            Setting.simpleString("cloud.azure.management.keystore.password", Property.NodeScope,
+                Property.Filtered);
+        public static final Setting<KeyStoreType> KEYSTORE_TYPE_SETTING =
+            new Setting<>("cloud.azure.management.keystore.type", KeyStoreType.pkcs12.name(), KeyStoreType::fromString,
+                Property.NodeScope, Property.Filtered);
     }
 
     final class Discovery {
-        public static final Setting<TimeValue> REFRESH_SETTING = Setting.positiveTimeSetting("discovery.azure.refresh_interval", TimeValue.timeValueSeconds(0), false, Setting.Scope.CLUSTER);
+        public static final Setting<TimeValue> REFRESH_SETTING =
+            Setting.positiveTimeSetting("discovery.azure.refresh_interval", TimeValue.timeValueSeconds(0), Property.NodeScope);
 
-        public static final Setting<AzureUnicastHostsProvider.HostType> HOST_TYPE_SETTING = new Setting<>("discovery.azure.host.type",
-            AzureUnicastHostsProvider.HostType.PRIVATE_IP.name(), AzureUnicastHostsProvider.HostType::fromString, false, Setting.Scope.CLUSTER);
+        public static final Setting<AzureUnicastHostsProvider.HostType> HOST_TYPE_SETTING =
+            new Setting<>("discovery.azure.host.type", AzureUnicastHostsProvider.HostType.PRIVATE_IP.name(),
+                AzureUnicastHostsProvider.HostType::fromString, Property.NodeScope);
 
         public static final String ENDPOINT_NAME = "discovery.azure.endpoint.name";
         public static final String DEPLOYMENT_NAME = "discovery.azure.deployment.name";
diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java b/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java
index b2b94b6c3bb..e5de7a0cb02 100644
--- a/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java
+++ b/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java
@@ -75,10 +75,5 @@ public class AzureDiscoveryPlugin extends Plugin {
         settingsModule.registerSetting(AzureComputeService.Management.SUBSCRIPTION_ID_SETTING);
         settingsModule.registerSetting(AzureComputeService.Management.SERVICE_NAME_SETTING);
         settingsModule.registerSetting(AzureComputeService.Discovery.HOST_TYPE_SETTING);
-        // Cloud management API settings we need to hide
-        settingsModule.registerSettingsFilter(AzureComputeService.Management.KEYSTORE_PATH_SETTING.getKey());
-        settingsModule.registerSettingsFilter(AzureComputeService.Management.KEYSTORE_PASSWORD_SETTING.getKey());
-        settingsModule.registerSettingsFilter(AzureComputeService.Management.KEYSTORE_TYPE_SETTING.getKey());
-        settingsModule.registerSettingsFilter(AzureComputeService.Management.SUBSCRIPTION_ID_SETTING.getKey());
     }
 }
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java
index a90d3573468..8cfe6c43108 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java
@@ -22,6 +22,7 @@ package org.elasticsearch.cloud.aws;
 import com.amazonaws.Protocol;
 import com.amazonaws.services.ec2.AmazonEC2;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 
@@ -32,7 +33,7 @@ import java.util.Locale;
 import java.util.function.Function;
 
 public interface AwsEc2Service {
-    Setting<Boolean> AUTO_ATTRIBUTE_SETTING = Setting.boolSetting("cloud.node.auto_attributes", false, false, Setting.Scope.CLUSTER);
+    Setting<Boolean> AUTO_ATTRIBUTE_SETTING = Setting.boolSetting("cloud.node.auto_attributes", false, Property.NodeScope);
 
     // Global AWS settings (shared between discovery-ec2 and repository-s3)
     // Each setting starting with `cloud.aws` also exists in repository-s3 project. Don't forget to update
@@ -40,40 +41,44 @@ public interface AwsEc2Service {
     /**
      * cloud.aws.access_key: AWS Access key. Shared with repository-s3 plugin
      */
-    Setting<String> KEY_SETTING = Setting.simpleString("cloud.aws.access_key", false, Setting.Scope.CLUSTER);
+    Setting<String> KEY_SETTING =
+        Setting.simpleString("cloud.aws.access_key", Property.NodeScope, Property.Filtered);
     /**
      * cloud.aws.secret_key: AWS Secret key. Shared with repository-s3 plugin
      */
-    Setting<String> SECRET_SETTING = Setting.simpleString("cloud.aws.secret_key", false, Setting.Scope.CLUSTER);
+    Setting<String> SECRET_SETTING =
+        Setting.simpleString("cloud.aws.secret_key", Property.NodeScope, Property.Filtered);
     /**
      * cloud.aws.protocol: Protocol for AWS API: http or https. Defaults to https. Shared with repository-s3 plugin
      */
     Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
-        false, Setting.Scope.CLUSTER);
+        Property.NodeScope);
     /**
      * cloud.aws.proxy.host: In case of proxy, define its hostname/IP. Shared with repository-s3 plugin
      */
-    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host", false, Setting.Scope.CLUSTER);
+    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host", Property.NodeScope);
     /**
      * cloud.aws.proxy.port: In case of proxy, define its port. Defaults to 80. Shared with repository-s3 plugin
      */
-    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16, false, Setting.Scope.CLUSTER);
+    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16, Property.NodeScope);
     /**
      * cloud.aws.proxy.username: In case of proxy with auth, define the username. Shared with repository-s3 plugin
      */
-    Setting<String> PROXY_USERNAME_SETTING = Setting.simpleString("cloud.aws.proxy.username", false, Setting.Scope.CLUSTER);
+    Setting<String> PROXY_USERNAME_SETTING = Setting.simpleString("cloud.aws.proxy.username", Property.NodeScope);
     /**
      * cloud.aws.proxy.password: In case of proxy with auth, define the password. Shared with repository-s3 plugin
      */
-    Setting<String> PROXY_PASSWORD_SETTING = Setting.simpleString("cloud.aws.proxy.password", false, Setting.Scope.CLUSTER);
+    Setting<String> PROXY_PASSWORD_SETTING =
+        Setting.simpleString("cloud.aws.proxy.password", Property.NodeScope, Property.Filtered);
     /**
      * cloud.aws.signer: If you are using an old AWS API version, you can define a Signer. Shared with repository-s3 plugin
      */
-    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", false, Setting.Scope.CLUSTER);
+    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", Property.NodeScope);
     /**
      * cloud.aws.region: Region. Shared with repository-s3 plugin
      */
-    Setting<String> REGION_SETTING = new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), false, Setting.Scope.CLUSTER);
+    Setting<String> REGION_SETTING =
+        new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
 
     /**
      * Defines specific ec2 settings starting with cloud.aws.ec2.
@@ -83,63 +88,63 @@ public interface AwsEc2Service {
          * cloud.aws.ec2.access_key: AWS Access key specific for EC2 API calls. Defaults to cloud.aws.access_key.
          * @see AwsEc2Service#KEY_SETTING
          */
-        Setting<String> KEY_SETTING = new Setting<>("cloud.aws.ec2.access_key", AwsEc2Service.KEY_SETTING, Function.identity(), false,
-            Setting.Scope.CLUSTER);
+        Setting<String> KEY_SETTING = new Setting<>("cloud.aws.ec2.access_key", AwsEc2Service.KEY_SETTING, Function.identity(),
+            Property.NodeScope, Property.Filtered);
         /**
          * cloud.aws.ec2.secret_key: AWS Secret key specific for EC2 API calls. Defaults to cloud.aws.secret_key.
          * @see AwsEc2Service#SECRET_SETTING
          */
-        Setting<String> SECRET_SETTING = new Setting<>("cloud.aws.ec2.secret_key", AwsEc2Service.SECRET_SETTING, Function.identity(), false,
-            Setting.Scope.CLUSTER);
+        Setting<String> SECRET_SETTING = new Setting<>("cloud.aws.ec2.secret_key", AwsEc2Service.SECRET_SETTING, Function.identity(),
+            Property.NodeScope, Property.Filtered);
         /**
          * cloud.aws.ec2.protocol: Protocol for AWS API specific for EC2 API calls: http or https.  Defaults to cloud.aws.protocol.
          * @see AwsEc2Service#PROTOCOL_SETTING
          */
         Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.ec2.protocol", AwsEc2Service.PROTOCOL_SETTING,
-            s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), false, Setting.Scope.CLUSTER);
+            s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope);
         /**
          * cloud.aws.ec2.proxy.host: In case of proxy, define its hostname/IP specific for EC2 API calls. Defaults to cloud.aws.proxy.host.
          * @see AwsEc2Service#PROXY_HOST_SETTING
          */
         Setting<String> PROXY_HOST_SETTING = new Setting<>("cloud.aws.ec2.proxy.host", AwsEc2Service.PROXY_HOST_SETTING,
-            Function.identity(), false, Setting.Scope.CLUSTER);
+            Function.identity(), Property.NodeScope);
         /**
          * cloud.aws.ec2.proxy.port: In case of proxy, define its port specific for EC2 API calls.  Defaults to cloud.aws.proxy.port.
          * @see AwsEc2Service#PROXY_PORT_SETTING
          */
         Setting<Integer> PROXY_PORT_SETTING = new Setting<>("cloud.aws.ec2.proxy.port", AwsEc2Service.PROXY_PORT_SETTING,
-            s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.ec2.proxy.port"), false, Setting.Scope.CLUSTER);
+            s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.ec2.proxy.port"), Property.NodeScope);
         /**
          * cloud.aws.ec2.proxy.username: In case of proxy with auth, define the username specific for EC2 API calls.
          * Defaults to cloud.aws.proxy.username.
          * @see AwsEc2Service#PROXY_USERNAME_SETTING
          */
         Setting<String> PROXY_USERNAME_SETTING = new Setting<>("cloud.aws.ec2.proxy.username", AwsEc2Service.PROXY_USERNAME_SETTING,
-            Function.identity(), false, Setting.Scope.CLUSTER);
+            Function.identity(), Property.NodeScope);
         /**
          * cloud.aws.ec2.proxy.password: In case of proxy with auth, define the password specific for EC2 API calls.
          * Defaults to cloud.aws.proxy.password.
          * @see AwsEc2Service#PROXY_PASSWORD_SETTING
          */
         Setting<String> PROXY_PASSWORD_SETTING = new Setting<>("cloud.aws.ec2.proxy.password", AwsEc2Service.PROXY_PASSWORD_SETTING,
-            Function.identity(), false, Setting.Scope.CLUSTER);
+            Function.identity(), Property.NodeScope, Property.Filtered);
         /**
          * cloud.aws.ec2.signer: If you are using an old AWS API version, you can define a Signer. Specific for EC2 API calls.
          * Defaults to cloud.aws.signer.
          * @see AwsEc2Service#SIGNER_SETTING
          */
         Setting<String> SIGNER_SETTING = new Setting<>("cloud.aws.ec2.signer", AwsEc2Service.SIGNER_SETTING, Function.identity(),
-            false, Setting.Scope.CLUSTER);
+            Property.NodeScope);
         /**
          * cloud.aws.ec2.region: Region specific for EC2 API calls. Defaults to cloud.aws.region.
          * @see AwsEc2Service#REGION_SETTING
          */
         Setting<String> REGION_SETTING = new Setting<>("cloud.aws.ec2.region", AwsEc2Service.REGION_SETTING,
-            s -> s.toLowerCase(Locale.ROOT), false, Setting.Scope.CLUSTER);
+            s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
         /**
          * cloud.aws.ec2.endpoint: Endpoint. If not set, endpoint will be guessed based on region setting.
          */
-        Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.ec2.endpoint", false, Setting.Scope.CLUSTER);
+        Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.ec2.endpoint", Property.NodeScope);
     }
 
     /**
@@ -158,32 +163,32 @@ public interface AwsEc2Service {
          * Can be one of private_ip, public_ip, private_dns, public_dns. Defaults to private_ip.
          */
         Setting<HostType> HOST_TYPE_SETTING =
-            new Setting<>("discovery.ec2.host_type", HostType.PRIVATE_IP.name(), s -> HostType.valueOf(s.toUpperCase(Locale.ROOT)), false,
-                Setting.Scope.CLUSTER);
+            new Setting<>("discovery.ec2.host_type", HostType.PRIVATE_IP.name(), s -> HostType.valueOf(s.toUpperCase(Locale.ROOT)),
+                Property.NodeScope);
         /**
          * discovery.ec2.any_group: If set to false, will require all security groups to be present for the instance to be used for the
          * discovery. Defaults to true.
          */
         Setting<Boolean> ANY_GROUP_SETTING =
-            Setting.boolSetting("discovery.ec2.any_group", true, false, Setting.Scope.CLUSTER);
+            Setting.boolSetting("discovery.ec2.any_group", true, Property.NodeScope);
         /**
          * discovery.ec2.groups: Either a comma separated list or array based list of (security) groups. Only instances with the provided
          * security groups will be used in the cluster discovery. (NOTE: You could provide either group NAME or group ID.)
          */
         Setting<List<String>> GROUPS_SETTING =
-            Setting.listSetting("discovery.ec2.groups", new ArrayList<>(), s -> s.toString(), false, Setting.Scope.CLUSTER);
+            Setting.listSetting("discovery.ec2.groups", new ArrayList<>(), s -> s.toString(), Property.NodeScope);
         /**
          * discovery.ec2.availability_zones: Either a comma separated list or array based list of availability zones. Only instances within
          * the provided availability zones will be used in the cluster discovery.
          */
         Setting<List<String>> AVAILABILITY_ZONES_SETTING =
-            Setting.listSetting("discovery.ec2.availability_zones", Collections.emptyList(), s -> s.toString(), false,
-                Setting.Scope.CLUSTER);
+            Setting.listSetting("discovery.ec2.availability_zones", Collections.emptyList(), s -> s.toString(),
+                Property.NodeScope);
         /**
          * discovery.ec2.node_cache_time: How long the list of hosts is cached to prevent further requests to the AWS API. Defaults to 10s.
          */
         Setting<TimeValue> NODE_CACHE_TIME_SETTING =
-            Setting.timeSetting("discovery.ec2.node_cache_time", TimeValue.timeValueSeconds(10), false, Setting.Scope.CLUSTER);
+            Setting.timeSetting("discovery.ec2.node_cache_time", TimeValue.timeValueSeconds(10), Property.NodeScope);
 
         /**
          * discovery.ec2.tag.*: The ec2 discovery can filter machines to include in the cluster based on tags (and not just groups).
@@ -191,7 +196,7 @@ public interface AwsEc2Service {
          * instances with a tag key set to stage, and a value of dev. Several tags set will require all of those tags to be set for the
          * instance to be included.
          */
-        Setting<Settings> TAG_SETTING = Setting.groupSetting("discovery.ec2.tag.", false,Setting.Scope.CLUSTER);
+        Setting<Settings> TAG_SETTING = Setting.groupSetting("discovery.ec2.tag.", Property.NodeScope);
     }
 
     AmazonEC2 client();
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java
index 36eae9b5829..a76a2b04a91 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java
@@ -55,7 +55,7 @@ public class AwsSigner {
         try {
             validateSignerType(signer);
         } catch (IllegalArgumentException e) {
-            logger.warn(e.getMessage());
+            logger.warn("{}", e.getMessage());
         }
 
         configuration.setSignerOverride(signer);
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
index 12d5682d487..b4c293ec736 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
@@ -134,14 +134,6 @@ public class Ec2DiscoveryPlugin extends Plugin {
         settingsModule.registerSetting(AwsEc2Service.DISCOVERY_EC2.GROUPS_SETTING);
         settingsModule.registerSetting(AwsEc2Service.DISCOVERY_EC2.AVAILABILITY_ZONES_SETTING);
         settingsModule.registerSetting(AwsEc2Service.DISCOVERY_EC2.NODE_CACHE_TIME_SETTING);
-
-        // Filter global settings
-        settingsModule.registerSettingsFilterIfMissing(AwsEc2Service.KEY_SETTING.getKey());
-        settingsModule.registerSettingsFilterIfMissing(AwsEc2Service.SECRET_SETTING.getKey());
-        settingsModule.registerSettingsFilterIfMissing(AwsEc2Service.PROXY_PASSWORD_SETTING.getKey());
-        settingsModule.registerSettingsFilterIfMissing(AwsEc2Service.CLOUD_EC2.KEY_SETTING.getKey());
-        settingsModule.registerSettingsFilterIfMissing(AwsEc2Service.CLOUD_EC2.SECRET_SETTING.getKey());
-        settingsModule.registerSettingsFilterIfMissing(AwsEc2Service.CLOUD_EC2.PROXY_PASSWORD_SETTING.getKey());
     }
 
     /**
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
index 5063d59b40e..1705421207b 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
@@ -191,7 +191,7 @@ public class Ec2DiscoveryTests extends ESTestCase {
             tagsList.add(tags);
         }
 
-        logger.info("started [{}] instances with [{}] stage=prod tag");
+        logger.info("started [{}] instances with [{}] stage=prod tag", nodes, prodInstances);
         List<DiscoveryNode> discoveryNodes = buildDynamicNodes(nodeSettings, nodes, tagsList);
         assertThat(discoveryNodes, hasSize(prodInstances));
     }
@@ -222,7 +222,7 @@ public class Ec2DiscoveryTests extends ESTestCase {
             tagsList.add(tags);
         }
 
-        logger.info("started [{}] instances with [{}] stage=prod tag");
+        logger.info("started [{}] instances with [{}] stage=prod tag", nodes, prodInstances);
         List<DiscoveryNode> discoveryNodes = buildDynamicNodes(nodeSettings, nodes, tagsList);
         assertThat(discoveryNodes, hasSize(prodInstances));
     }
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java
index ce5154b3436..a6faa390e5d 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java
@@ -22,12 +22,14 @@ package org.elasticsearch.cloud.gce;
 import com.google.api.services.compute.model.Instance;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.TimeValue;
 
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.function.Function;
 
 public interface GceComputeService extends LifecycleComponent<GceComputeService> {
 
@@ -41,25 +43,25 @@ public interface GceComputeService extends LifecycleComponent<GceComputeService>
     /**
      * cloud.gce.project_id: Google project id
      */
-    Setting<String> PROJECT_SETTING = Setting.simpleString("cloud.gce.project_id", false, Setting.Scope.CLUSTER);
+    Setting<String> PROJECT_SETTING = Setting.simpleString("cloud.gce.project_id", Property.NodeScope);
 
     /**
      * cloud.gce.zone: Google Compute Engine zones
      */
     Setting<List<String>> ZONE_SETTING =
-        Setting.listSetting("cloud.gce.zone", Collections.emptyList(), s -> s, false, Setting.Scope.CLUSTER);
+        Setting.listSetting("cloud.gce.zone", Collections.emptyList(), Function.identity(), Property.NodeScope);
 
     /**
      * cloud.gce.refresh_interval: How long the list of hosts is cached to prevent further requests to the AWS API. 0 disables caching.
      * A negative value will cause infinite caching. Defaults to 0s.
      */
     Setting<TimeValue> REFRESH_SETTING =
-        Setting.timeSetting("cloud.gce.refresh_interval", TimeValue.timeValueSeconds(0), false, Setting.Scope.CLUSTER);
+        Setting.timeSetting("cloud.gce.refresh_interval", TimeValue.timeValueSeconds(0), Property.NodeScope);
 
     /**
      * cloud.gce.retry: Should we retry calling GCE API in case of error? Defaults to true.
      */
-    Setting<Boolean> RETRY_SETTING = Setting.boolSetting("cloud.gce.retry", true, false, Setting.Scope.CLUSTER);
+    Setting<Boolean> RETRY_SETTING = Setting.boolSetting("cloud.gce.retry", true, Property.NodeScope);
 
     /**
      * cloud.gce.max_wait: How long exponential backoff should retry before definitely failing.
@@ -67,7 +69,7 @@ public interface GceComputeService extends LifecycleComponent<GceComputeService>
      * A negative value will retry indefinitely. Defaults to `-1s` (retry indefinitely).
      */
     Setting<TimeValue> MAX_WAIT_SETTING =
-        Setting.timeSetting("cloud.gce.max_wait", TimeValue.timeValueSeconds(-1), false, Setting.Scope.CLUSTER);
+        Setting.timeSetting("cloud.gce.max_wait", TimeValue.timeValueSeconds(-1), Property.NodeScope);
 
     /**
      * Return a collection of running instances within the same GCE project
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java
index d9033b602d2..85e0910736f 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java
@@ -38,13 +38,13 @@ import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.gce.RetryHttpInitializerWrapper;
 
 import java.io.IOException;
 import java.net.URL;
-import java.nio.file.Files;
 import java.security.AccessController;
 import java.security.GeneralSecurityException;
 import java.security.PrivilegedAction;
@@ -61,11 +61,11 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent<GceCompute
 
     // all settings just used for testing - not registered by default
     public static final Setting<Boolean> GCE_VALIDATE_CERTIFICATES =
-        Setting.boolSetting("cloud.gce.validate_certificates", true, false, Setting.Scope.CLUSTER);
+        Setting.boolSetting("cloud.gce.validate_certificates", true, Property.NodeScope);
     public static final Setting<String> GCE_HOST =
-        new Setting<>("cloud.gce.host", "http://metadata.google.internal", Function.identity(), false, Setting.Scope.CLUSTER);
+        new Setting<>("cloud.gce.host", "http://metadata.google.internal", Function.identity(), Property.NodeScope);
     public static final Setting<String> GCE_ROOT_URL =
-        new Setting<>("cloud.gce.root_url", "https://www.googleapis.com", Function.identity(), false, Setting.Scope.CLUSTER);
+        new Setting<>("cloud.gce.root_url", "https://www.googleapis.com", Function.identity(), Property.NodeScope);
 
     private final String project;
     private final List<String> zones;
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java
index 4e7956c379e..85f3e3a9585 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java
@@ -31,6 +31,7 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
@@ -43,6 +44,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.function.Function;
 
 /**
  *
@@ -53,7 +55,7 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas
      * discovery.gce.tags: The gce discovery can filter machines to include in the cluster based on tags.
      */
     public static final Setting<List<String>> TAGS_SETTING =
-            Setting.listSetting("discovery.gce.tags", Collections.emptyList(), s -> s, false, Setting.Scope.CLUSTER);
+            Setting.listSetting("discovery.gce.tags", Collections.emptyList(), Function.identity(), Property.NodeScope);
 
     static final class Status {
         private static final String TERMINATED = "TERMINATED";
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
index d355276ceae..2b92c4fd8c1 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
@@ -47,7 +47,7 @@ public class GceDiscoveryPlugin extends Plugin {
     static {
         /*
          * GCE's http client changes access levels because its silly and we
-         * can't allow that on any old stack stack so we pull it here, up front,
+         * can't allow that on any old stack so we pull it here, up front,
          * so we can cleanly check the permissions for it. Without this changing
          * the permission can fail if any part of core is on the stack because
          * our plugin permissions don't allow core to "reach through" plugins to
diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java
index 2ea977b4dd1..27a4cfebbcd 100644
--- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java
+++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java
@@ -137,6 +137,8 @@ final class TikaImpl {
         perms.add(new SecurityPermission("putProviderProperty.BC"));
         perms.add(new SecurityPermission("insertProvider"));
         perms.add(new ReflectPermission("suppressAccessChecks"));
+        // xmlbeans, use by POI, needs to get the context classloader
+        perms.add(new RuntimePermission("getClassLoader"));
         perms.setReadOnly();
         return perms;
     }
diff --git a/plugins/ingest-attachment/src/main/plugin-metadata/plugin-security.policy b/plugins/ingest-attachment/src/main/plugin-metadata/plugin-security.policy
index e23e9f4d0cf..adf76991b59 100644
--- a/plugins/ingest-attachment/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/ingest-attachment/src/main/plugin-metadata/plugin-security.policy
@@ -27,4 +27,6 @@ grant {
   permission java.security.SecurityPermission "insertProvider";
   // TODO: fix POI XWPF to not do this: https://bz.apache.org/bugzilla/show_bug.cgi?id=58597
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+  // needed by xmlbeans, as part of POI for MS xml docs
+  permission java.lang.RuntimePermission "getClassLoader";
 };
diff --git a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/10_basic.yaml b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/10_basic.yaml
index 67bb7340ce3..7c789b9c2ca 100644
--- a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/10_basic.yaml
+++ b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/10_basic.yaml
@@ -8,4 +8,19 @@
         nodes.info: {}
 
     - match:  { nodes.$master.plugins.0.name: ingest-attachment  }
-    - match:  { nodes.$master.ingest.processors.11.type: attachment }
+    - match:  { nodes.$master.ingest.processors.0.type: append }
+    - match:  { nodes.$master.ingest.processors.1.type: attachment }
+    - match:  { nodes.$master.ingest.processors.2.type: convert }
+    - match:  { nodes.$master.ingest.processors.3.type: date }
+    - match:  { nodes.$master.ingest.processors.4.type: fail }
+    - match:  { nodes.$master.ingest.processors.5.type: foreach }
+    - match:  { nodes.$master.ingest.processors.6.type: gsub }
+    - match:  { nodes.$master.ingest.processors.7.type: join }
+    - match:  { nodes.$master.ingest.processors.8.type: lowercase }
+    - match:  { nodes.$master.ingest.processors.9.type: remove }
+    - match:  { nodes.$master.ingest.processors.10.type: rename }
+    - match:  { nodes.$master.ingest.processors.11.type: set }
+    - match:  { nodes.$master.ingest.processors.12.type: split }
+    - match:  { nodes.$master.ingest.processors.13.type: trim }
+    - match:  { nodes.$master.ingest.processors.14.type: uppercase }
+
diff --git a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yaml b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yaml
new file mode 100644
index 00000000000..4f56603a678
--- /dev/null
+++ b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/30_files_supported.yaml
@@ -0,0 +1,79 @@
+---
+"Test ingest attachment processor with .doc file":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "attachment" : {
+                  "source_field" : "field1"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: { field1: "0M8R4KGxGuEAAAAAAAAAAAAAAAAAAAAAPgADAP7/CQAGAAAAAAAAAAAAAAAEAAAAjAEAAAAAAAAAEAAAjgEAAAEAAAD+////AAAAAIgBAACJAQAAigEAAIsBAAD////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////spcEAg+kMBAAA8BK/AAAAAAABEQABAAEACAAAEwgAAA4AYmpiaoI4gjgAAAAAAAAAAAAAAAAAAAAAAAAMBBYANA4AAOBSAADgUgAAEwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD//w8AAAAAAAAAAAD//w8AAAAAAAAAAAD//w8AAAAAAAAAAAAAAAAAAAAAALcAAAAAAFAHAAAAAAAAUAcAAMcUAAAAAAAAxxQAAAAAAADHFAAAAAAAAMcUAAAAAAAAxxQAABQAAAAAAAAAAAAAAP////8AAAAA2xQAAAAAAADbFAAAAAAAANsUAAAAAAAA2xQAAAwAAADnFAAADAAAANsUAAAAAAAA3hUAADABAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAAVRUAAAIAAABXFQAAAAAAAFcVAAAAAAAAVxUAAAAAAABXFQAAAAAAAFcVAAAAAAAAVxUAACwAAAAOFwAAtgIAAMQZAABaAAAAgxUAABUAAAAAAAAAAAAAAAAAAAAAAAAAxxQAAAAAAADzFAAAAAAAAAAAAAAAAAAAAAAAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAIMVAAAAAAAAGRUAAAAAAADHFAAAAAAAAMcUAAAAAAAA8xQAAAAAAAAAAAAAAAAAAPMUAAAAAAAAmBUAABYAAAAZFQAAAAAAABkVAAAAAAAAGRUAAAAAAADzFAAAFgAAAMcUAAAAAAAA8xQAAAAAAADHFAAAAAAAAPMUAAAAAAAAVRUAAAAAAAAAAAAAAAAAABkVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA8xQAAAAAAABVFQAAAAAAAAAAAAAAAAAAGRUAAAAAAAAAAAAAAAAAABkVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRUAAAAAAAAAAAAAAAAAAP////8AAAAAgI6XYKZ60QEAAAAAAAAAAP////8AAAAACRUAABAAAAAZFQAAAAAAAAAAAAAAAAAAQRUAABQAAACuFQAAMAAAAN4VAAAAAAAAGRUAAAAAAAAeGgAAAAAAABkVAAAAAAAAHhoAAAAAAAAZFQAAAAAAABkVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADHFAAAAAAAABkVAAAoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAAgxUAAAAAAACDFQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAN4VAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAAAAAAAAAAAAAP////8AAAAA/////wAAAAD/////AAAAAAAAAAAAAAAA/////wAAAAD/////AAAAAP////8AAAAA/////wAAAAD/////AAAAAP////8AAAAA/////wAAAAD/////AAAAAP////8AAAAA/////wAAAAD/////AAAAAP////8AAAAA/////wAAAAD/////AAAAAB4aAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAABQBwAAPQwAAI0TAAA6AQAABwAMAQ8ADQEAAAwEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFRlc3QgZWxhc3RpY3NlYXJjaA0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAABIIAAATCAAA/PgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYWaJVGuQAABhZo3wiGAAIACAAAEwgAAP0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAATIAMZBoATpwpBeqAB+wfC4gsMhBIbCJBSKwiQUjkIkFJJCJBSWwAAAXsMQCGLDEAgyQxAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALgYPABIAAQB8AQ8ACAADAAMAAwAAAAQACAAAAJgAAACeAAAAngAAAJ4AAACeAAAAngAAAJ4AAACeAAAAngAAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAAHYCAAB2AgAAdgIAAHYCAAB2AgAAdgIAAHYCAAB2AgAAdgIAADYGAAA2BgAANgYAADYGAAA2BgAANgYAAD4CAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAACoAAAANgYAADYGAAAWAAAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAC4AAAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAAaAEAAEgBAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAAHACAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAAMgYAABgAAADGAwAA1gMAAOYDAAD2AwAABgQAABYEAAAmBAAANgQAAEYEAABWBAAAZgQAAHYEAACGBAAAlgQAAMYDAADWAwAA5gMAAPYDAAAGBAAAFgQAADIGAAAoAgAA2AEAAOgBAAAmBAAANgQAAEYEAABWBAAAZgQAAHYEAACGBAAAlgQAAMYDAADWAwAA5gMAAPYDAAAGBAAAFgQAACYEAAA2BAAARgQAAFYEAABmBAAAdgQAAIYEAACWBAAAxgMAANYDAADmAwAA9gMAAAYEAAAWBAAAJgQAADYEAABGBAAAVgQAAGYEAAB2BAAAhgQAAJYEAADGAwAA1gMAAOYDAAD2AwAABgQAABYEAAAmBAAANgQAAEYEAABWBAAAZgQAAHYEAACGBAAAlgQAAMYDAADWAwAA5gMAAPYDAAAGBAAAFgQAACYEAAA2BAAARgQAAFYEAABmBAAAdgQAAIYEAACWBAAAxgMAANYDAADmAwAA9gMAAAYEAAAWBAAAJgQAADYEAABGBAAAVgQAAGYEAAB2BAAAhgQAAJYEAAA4AQAAWAEAAPgBAAAIAgAAGAIAAFYCAAB+AgAAkAIAAKACAACwAgAAwAIAANACAACAAgAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAADgAgAA8AIAAAADAAAQAwAAIAMAADADAABAAwAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAADgAgAA8AIAAAADAAAQAwAAIAMAADADAABAAwAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAADgAgAA8AIAAAADAAAQAwAAIAMAADADAABAAwAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAADgAgAA8AIAAAADAAAQAwAAIAMAADADAABAAwAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAAAgAAAAT0oDAFBKAwBRSgMAX0gBBG1IDARuSAwEc0gMBHRIDAQAAAAAQAAAYPH/AgBAAAwQAAAAAAAAAAAGAE4AbwByAG0AYQBsAAAAAgAAABgAQ0oYAF9IAQRhShgAbUgMBHNIDAR0SAkEAAAAAAAAAAAAAAAAAAAAAAAAOgBBIPL/oQA6AAwNAAAAAAAAEAARAFAAbwBsAGkAYwBlACAAcABhAHIAIABkAOkAZgBhAHUAdAAAAAAAVgBpAPP/swBWAAwNAAAAAAAAMAYOAFQAYQBiAGwAZQBhAHUAIABOAG8AcgBtAGEAbAAAABwAF/YDAAA01gYAAQoDbAA01gYAAQUDAABh9gMAAAIACwAAADIAayD0/8EAMgAADQAAAAAAADAGDABBAHUAYwB1AG4AZQAgAGwAaQBzAHQAZQAAAAIADAAAAAAAUEsDBBQABgAIAAAAIQCb6HBP/AAAABwCAAATAAAAW0NvbnRlbnRfVHlwZXNdLnhtbKyRy2rDMBBF94X+g9C22HK6KKXYzqKPXR+L9AMGeWyL2CMhTULy9x07LpQSAoVuBNLMvffMqFwfxkHtMSbnqdKrvNAKyfrGUVfpz81Ldq9VYqAGBk9Y6SMmva6vr8rNMWBSoqZU6Z45PBiTbI8jpNwHJKm0Po7Aco2dCWC30KG5LYo7Yz0xEmc8eei6fMIWdgOr54M8n0hErtXjqW+KqjSEMDgLLKBmqpqzuohDuiDcU/OLLlvIclHO5ql3Id0sCe+ymugaVB8Q+Q1G4TAsQ+LP8xVIRov5ZeYz0b5tncXG290o68hn48XsTwCr/4n+zjTz39ZfAAAA//8DAFBLAwQUAAYACAAAACEApdan58AAAAA2AQAACwAAAF9yZWxzLy5yZWxzhI/PasMwDIfvhb2D0X1R0sMYJXYvpZBDL6N9AOEof2giG9sb69tPxwYKuwiEpO/3qT3+rov54ZTnIBaaqgbD4kM/y2jhdj2/f4LJhaSnJQhbeHCGo3vbtV+8UNGjPM0xG6VItjCVEg+I2U+8Uq5CZNHJENJKRds0YiR/p5FxX9cfmJ4Z4DZM0/UWUtc3YK6PqMn/s8MwzJ5PwX+vLOVFBG43lExp5GKhqC/jU72QqGWq1B7Qtbj51v0BAAD//wMAUEsDBBQABgAIAAAAIQBreZYWgwAAAIoAAAAcAAAAdGhlbWUvdGhlbWUvdGhlbWVNYW5hZ2VyLnhtbAzMTQrDIBBA4X2hd5DZN2O7KEVissuuu/YAQ5waQceg0p/b1+XjgzfO3xTVm0sNWSycBw2KZc0uiLfwfCynG6jaSBzFLGzhxxXm6XgYybSNE99JyHNRfSPVkIWttd0g1rUr1SHvLN1euSRqPYtHV+jT9yniResrJgoCOP0BAAD//wMAUEsDBBQABgAIAAAAIQBtTVmryAYAAI4aAAAWAAAAdGhlbWUvdGhlbWUvdGhlbWUxLnhtbOxZ3YrbRhS+L/QdhO4d/0n+WeINtmxv2uwmIXbS5nJWHkuTHWmMZrwbEwJ9gkIhLb0p9K6F3gTaN+i7pLTpQ/TMSJZn7HH2hy2E0jUs8vg7Z7455+g7I83dey8T6pzjjBOW9tz6nZrr4DRkM5JGPffpdFzpuA4XKJ0hylLcc1eYu/cOP/3kLjoQMU6wA/YpP0A9NxZicVCt8hCGEb/DFjiF3+YsS5CAr1lUnWXoAvwmtNqo1VrVBJHUdVKUgNtp/PvP4OzRfE5C7B6uvY8oTJEKLgdCmk2kb1yYDJYZRkuFnZ3VJYKveEAz5xzRngsTzdjFFL8UrkMRF/BDz62pP7d6eLeKDgojKvbYanZj9VfYFQazs4aaM4tOy0k9z/da/dK/AlCxixu1R61Rq/SnACgMYaU5F92nP+gOhn6B1UD5pcX3sD1s1g285r+5w7nvy4+BV6Dcv7eDH48DiKKBV6Ac7+/gPa/dCDwDr0A5vrWDb9f6Q69t4BUopiQ920HX/FYzWK+2hMwZvW+Fd31v3G4UzjcoqIayuuQUc5aKfbWWoBcsGwNAAikSJHXEaoHnKIQyDhAlpxlxjkkUQ+EtUMo4DNcatXGtCf/lx1NXKiLoACPNWvICJnxnSPJxeJiRhei5n4NXV4M8XzpHTMQkLGZVTgyL+yiNdIv3P33z9w9fOX/9+uP7N9/mk27juY4f4jT6kqD0QxPAajdhePfd2z9+e/vu+6///OWNxX8/Q6c6fEoSzJ2H+MJ5whJYnGUF+DS7nsU0RkS36KcRRymSs1j8jyB+OvrhClFkwQ0gEjruWQYyYwMeLV8YhCdxthTE4vFBnBjAE8bogGXWKDyQc2lhni7TyD55ttRxTxA6t80doNTI82i5AH0lNpdBjA2ajylKBYpwioUjf2NnGFtW95wQI64nJMwYZ3PhPCfOABFrSKbk1KimjdF9kkBeVjaCkG8jNifPnAGjtlUP8bmJhLsDUQv5KaZGGI/QUqDE5nKKEqoH/BiJ2EZysspCHTfiAjIdYcqc0QxzbrN5lMF6taQ/AImxp/2ErhITmQlyZvN5jBjTkUN2FsQoWdiwE5LGOvYzfgYlipzHTNjgJ8y8Q+R3yAOIx750PyPYSPflavAU1FWntCkQ+csys+TyCDOjficrOkdYSQ2Iv6HpCUkvFfgtaff/PWk/IWkYM8uKbkvU7a6NjFxTzvsZsd5P97dEfB9uW7oDls3Ix6/cQ7RMH2O4WXbb1//C/b9wu/954d53P9++XG8UGsRbbl3zzbrauid7d+5zQulErCg+5mrzzqEvzcYwKO3UYysun+QWMVzKOxkmMHBRhpSNkzHxBRHxJEYL2OHXXekk4oXriDsLxmHjr4atviWeLpMTNssfWOt1+XCaiwdHYjNe88txeNgQObrV3jyEle4V20g9LK8JSNvrkNAmM0k0LSTa60EZJPVoDkGzkFAruxUWXQuLjnS/TtUOC6BWZgU2Tg5st3qu74EJGMEzFaJ4JvOUp3qdXZXM28z0vmAaFQC7iHUFbDLdlVz3Lk+uLi+1K2TaIKGVm0lCRUb1MB6jGS6qU45ehcZ1c93dpNSgJ0Oh5oPS2tBodz7E4qa5BrttbaCprhQ0dS56bqvpQ8mEaNFz5/DgD5fJAmqHyw0vohG8PgtFlt/wN1GWRcbFEPE4D7gSnVwNEiJw5lCS9Fy5/DINNFUaorjVGyAIHy25LsjKx0YOkm4mGc/nOBR62rURGen8Kyh8rhXWX5X5zcHSki0h3ZN4duGc0mX2BEGJ+e26DOCMcHj/U8+jOSPwQrMUsk39bTWmQnb1N4qqhvJxRBcxKjqKLuY5XEl5SUd9K2OgfSvWDAHVQlI0wtNINlg9qEY3LbtGzmFv173cSEZOE81NzzRURXZNu4oZM6zbwFYsb9bkNVbrEIOm6R0+l+5tye2utW5rn1B2CQh4GT9L171CQ9CobSYzqEnGuzIsNbsYNXvHeoGXULtKk9BUv7V2uxW3skdYp4PBG3V+sNuuWhiar/eVKtLq6EM/nGCnL0A8hvAaeEkFV6mEo4cMwYZoovYkuWzALfJSFLcGXDnLjPTcVzW/7wUNP6jUOv6o4jW9WqXj95uVvu836yO/XhsOGq+hsYg4qfv5scsYXkTRVXH4osZ3DmCS9bu2OyFLqkydrFQVcXUAU28YBzD5yYszlQcsrkNAdF61GuNusztoVbrN/rjiDQedSjdoDSrDVtAejoeB3+mOX7vOuQJ7/WbgtUadSqseBBWvVZP0O91K22s0+l673xl5/dfFNgZWnstHEQsIr+J1+A8AAAD//wMAUEsDBBQABgAIAAAAIQAN0ZCftgAAABsBAAAnAAAAdGhlbWUvdGhlbWUvX3JlbHMvdGhlbWVNYW5hZ2VyLnhtbC5yZWxzhI9NCsIwFIT3gncIb2/TuhCRJt2I0K3UA4TkNQ02PyRR7O0NriwILodhvplpu5edyRNjMt4xaKoaCDrplXGawW247I5AUhZOidk7ZLBggo5vN+0VZ5FLKE0mJFIoLjGYcg4nSpOc0IpU+YCuOKOPVuQio6ZByLvQSPd1faDxmwF8xSS9YhB71QAZllCa/7P9OBqJZy8fFl3+UUFz2YUFKKLGzOAjm6pMBMpburrE3wAAAP//AwBQSwECLQAUAAYACAAAACEAm+hwT/wAAAAcAgAAEwAAAAAAAAAAAAAAAAAAAAAAW0NvbnRlbnRfVHlwZXNdLnhtbFBLAQItABQABgAIAAAAIQCl1qfnwAAAADYBAAALAAAAAAAAAAAAAAAAAC0BAABfcmVscy8ucmVsc1BLAQItABQABgAIAAAAIQBreZYWgwAAAIoAAAAcAAAAAAAAAAAAAAAAABYCAAB0aGVtZS90aGVtZS90aGVtZU1hbmFnZXIueG1sUEsBAi0AFAAGAAgAAAAhAG1NWavIBgAAjhoAABYAAAAAAAAAAAAAAAAA0wIAAHRoZW1lL3RoZW1lL3RoZW1lMS54bWxQSwECLQAUAAYACAAAACEADdGQn7YAAAAbAQAAJwAAAAAAAAAAAAAAAADPCQAAdGhlbWUvdGhlbWUvX3JlbHMvdGhlbWVNYW5hZ2VyLnhtbC5yZWxzUEsFBgAAAAAFAAUAXQEAAMoKAAAAADw/eG1sIHZlcnNpb249IjEuMCIgZW5jb2Rpbmc9IlVURi04IiBzdGFuZGFsb25lPSJ5ZXMiPz4NCjxhOmNsck1hcCB4bWxuczphPSJodHRwOi8vc2NoZW1hcy5vcGVueG1sZm9ybWF0cy5vcmcvZHJhd2luZ21sLzIwMDYvbWFpbiIgYmcxPSJsdDEiIHR4MT0iZGsxIiBiZzI9Imx0MiIgdHgyPSJkazIiIGFjY2VudDE9ImFjY2VudDEiIGFjY2VudDI9ImFjY2VudDIiIGFjY2VudDM9ImFjY2VudDMiIGFjY2VudDQ9ImFjY2VudDQiIGFjY2VudDU9ImFjY2VudDUiIGFjY2VudDY9ImFjY2VudDYiIGhsaW5rPSJobGluayIgZm9sSGxpbms9ImZvbEhsaW5rIi8+AAAAABMAAAAUAAAOAAAIAP////8ACAAAEwgAAAUAAAAACAAAEwgAAAYAAAAAAAAABQAAABIAAAAVAAAABwAEAAcAAAAAABIAAAAVAAAABAAHAAQAAAAEAAAACAAAAOUAAAAAAAAAAwAAAN8IhgCkF6oAlUa5AH419AAAAAAAEwAAABUAAAAAAAAAAQAAAP9AAIABABIAAAASAAAAAEBDewEAAQASAAAAAAAAABIAAAAAAAAAAAAAAAAAAAACEAAAAAAAAAATAAAAoAAAEABAAAD//wEAAAAHAFUAbgBrAG4AbwB3AG4A//8BAAgAAAAAAAAAAAAAAP//AQAAAAAA//8AAAIA//8AAAAA//8AAAIA//8AAAAABQAAAEcOkAEAAAICBgMFBAUCAwTvKgDgQXgAwAkAAAAAAAAA/wEAAAAAAABUAGkAbQBlAHMAIABOAGUAdwAgAFIAbwBtAGEAbgAAADUOkAECAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAgAAAAABTAHkAbQBiAG8AbAAAADMOkAEAAAILBgQCAgICAgT/KgDgQ3gAwAkAAAAAAAAA/wEAAAAAAABBAHIAaQBhAGwAAAA3DpABAAACDwUCAgIEAwIE/wIA4P+sAEABAAAAAAAAAJ8BAAAAAAAAQwBhAGwAaQBiAHIAaQAAAEESkAEBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABDAGEAbQBiAHIAaQBhACAATQBhAHQAaAAAACAABADxCIgIAPDEAgAAqQEAAAAAWVJDh1lSQ4cAAAAAAgABAAAAAgAAABEAAAABAAEAAAAEAAOQAQAAAAIAAAARAAAAAQABAAAAAQAAAAAAAAAhAwDwEAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAClBsAHtAC0AIGBcjAAAAAAAAAAAAAAAAAAABIAAAASAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAABAAAAA8BAACAD8/QEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACSFAAAAAACfH/DwAAJFAAABAnAAD///9/////f////3////9/////f////3////9/3wiGAAAEAAAyAAAAAAAAAAAAAAAAAAAAAAAAAAAAIQQAAAAAAAAAAAAAAAAAAAAAAAAQHAAABAAAAAAAAAAAAHgAAAB4AAAAAAAAAAAAAACgBQAAGkjOCAsAAAAAAAAA3AAAAAEAAAD//xIAAAAAAAAAAAAAAAAAAAAMAEQAYQB2AGkAZAAgAFAAaQBsAGEAdABvAAwARABhAHYAaQBkACAAUABpAGwAYQB0AG8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP7/AAADCgEAAAAAAAAAAAAAAAAAAAAAAAEAAADghZ/y+U9oEKuRCAArJ7PZMAAAANzSAgASAAAAAQAAAJgAAAACAAAAoAAAAAMAAACsAAAABAAAALgAAAAFAAAA0AAAAAYAAADcAAAABwAAAOgAAAAIAAAA/AAAAAkAAAAUAQAAEgAAACABAAAKAAAARAEAAAwAAABQAQAADQAAAFwBAAAOAAAAaAEAAA8AAABwAQAAEAAAAHgBAAATAAAAgAEAABEAAACIAQAAAgAAABAnAAAeAAAABAAAAAAAAAAeAAAABAAAAAAAAAAeAAAAEAAAAERhdmlkIFBpbGF0bwAAAAAeAAAABAAAAAAAAAAeAAAABAAAAAAAAAAeAAAADAAAAE5vcm1hbC5kb3RtAB4AAAAQAAAARGF2aWQgUGlsYXRvAAAAAB4AAAAEAAAAMgAAAB4AAAAcAAAATWljcm9zb2Z0IE1hY2ludG9zaCBXb3JkAAAAAEAAAAAARsMjAAAAAEAAAAAAFjZWpnrRAUAAAAAAFjZWpnrRAQMAAAABAAAAAwAAAAIAAAADAAAAEQAAAAMAAAAAAAAARwAAAEzRAgD/////DgAAAAEAAABsAAAAAAAAAAAAAAD/AAAAswAAAAAAAAAAAAAAZhkAANsRAAAgRU1GAAABAETRAgAIAAAAAQAAAAAAAAAAAAAAAAAAAOwEAACxAwAAQAEAAPAAAAAAAAAAAAAAAAAAAAAA4gQAgKkDABEAAAAMAAAACAAAAAoAAAAQAAAAAAAAAAAAAAAJAAAAEAAAAAABAAC0AAAADAAAABAAAAAAAAAAAAAAAAsAAAAQAAAAAAEAALQAAABRAAAAeNACAAAAAAAAAAAA/wAAALMAAAAAAAAAAAAAAAAAAAAAAAAAAAEAALQAAABQAAAAKAAAAHgAAAAA0AIAAAAAACAAzAAAAQAAtAAAACgAAAAAAQAAtAAAAAEAIAAAAAAAANACAAAAAAAAAAAAAAAAAAAAAAD/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////vr6+/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/76+vv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////7vf//+rz7v/Yzc3/0NLY/+DX2f/N4PL/3tXI/8jV4v/Q0cX/1tDI/9ve2f/U0tX/0NLQ/83I0P/I2N7/4tnI/9LZ4v/v6tz/5eXl////9////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////83g9//e3M3/vrG3/8TCxv/Xwrz/vdfu/8W/rv/K1tX/x8bB/8LJxv/Oxb7/yMTE/8vCwv+3scH/zd7Z/9DNyP/BwcT/z97X/82xq/////v////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////u9/v/+/Lu////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////++vr7/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/vr6+//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8OAAAAFAAAAAAAAAAQAAAAFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD+/wAAAwoBAAAAAAAAAAAAAAAAAAAAAAABAAAAAtXN1ZwuGxCTlwgAKyz5rjAAAADUAAAACwAAAAEAAABgAAAABQAAAGgAAAAGAAAAcAAAABEAAAB4AAAAFwAAAIAAAAALAAAAiAAAABAAAACQAAAAEwAAAJgAAAAWAAAAoAAAAA0AAACoAAAADAAAALUAAAACAAAAECcAAAMAAAABAAAAAwAAAAEAAAADAAAAEgAAAAMAAAAAAA8ACwAAAAAAAAALAAAAAAAAAAsAAAAAAAAACwAAAAAAAAAeEAAAAQAAAAEAAAAADBAAAAIAAAAeAAAABgAAAFRpdHJlAAMAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAIAAAADAAAABAAAAAUAAAAGAAAABwAAAP7///8JAAAACgAAAAsAAAAMAAAADQAAAA4AAAAPAAAAEAAAABEAAAASAAAAEwAAABQAAAAVAAAA/v///xcAAAAYAAAAGQAAABoAAAAbAAAAHAAAAB0AAAAeAAAAHwAAACAAAAAhAAAAIgAAACMAAAAkAAAAJQAAACYAAAAnAAAAKAAAACkAAAAqAAAAKwAAACwAAAAtAAAALgAAAC8AAAAwAAAAMQAAADIAAAAzAAAANAAAADUAAAA2AAAANwAAADgAAAA5AAAAOgAAADsAAAA8AAAAPQAAAD4AAAA/AAAAQAAAAEEAAABCAAAAQwAAAEQAAABFAAAARgAAAEcAAABIAAAASQAAAEoAAABLAAAATAAAAE0AAABOAAAATwAAAFAAAABRAAAAUgAAAFMAAABUAAAAVQAAAFYAAABXAAAAWAAAAFkAAABaAAAAWwAAAFwAAABdAAAAXgAAAF8AAABgAAAAYQAAAGIAAABjAAAAZAAAAGUAAABmAAAAZwAAAGgAAABpAAAAagAAAGsAAABsAAAAbQAAAG4AAABvAAAAcAAAAHEAAAByAAAAcwAAAHQAAAB1AAAAdgAAAHcAAAB4AAAAeQAAAHoAAAB7AAAAfAAAAH0AAAB+AAAAfwAAAIAAAACBAAAAggAAAIMAAACEAAAAhQAAAIYAAACHAAAAiAAAAIkAAACKAAAAiwAAAIwAAACNAAAAjgAAAI8AAACQAAAAkQAAAJIAAACTAAAAlAAAAJUAAACWAAAAlwAAAJgAAACZAAAAmgAAAJsAAACcAAAAnQAAAJ4AAACfAAAAoAAAAKEAAACiAAAAowAAAKQAAAClAAAApgAAAKcAAACoAAAAqQAAAKoAAACrAAAArAAAAK0AAACuAAAArwAAALAAAACxAAAAsgAAALMAAAC0AAAAtQAAALYAAAC3AAAAuAAAALkAAAC6AAAAuwAAALwAAAC9AAAAvgAAAL8AAADAAAAAwQAAAMIAAADDAAAAxAAAAMUAAADGAAAAxwAAAMgAAADJAAAAygAAAMsAAADMAAAAzQAAAM4AAADPAAAA0AAAANEAAADSAAAA0wAAANQAAADVAAAA1gAAANcAAADYAAAA2QAAANoAAADbAAAA3AAAAN0AAADeAAAA3wAAAOAAAADhAAAA4gAAAOMAAADkAAAA5QAAAOYAAADnAAAA6AAAAOkAAADqAAAA6wAAAOwAAADtAAAA7gAAAO8AAADwAAAA8QAAAPIAAADzAAAA9AAAAPUAAAD2AAAA9wAAAPgAAAD5AAAA+gAAAPsAAAD8AAAA/QAAAP4AAAD/AAAAAAEAAAEBAAACAQAAAwEAAAQBAAAFAQAABgEAAAcBAAAIAQAACQEAAAoBAAALAQAADAEAAA0BAAAOAQAADwEAABABAAARAQAAEgEAABMBAAAUAQAAFQEAABYBAAAXAQAAGAEAABkBAAAaAQAAGwEAABwBAAAdAQAAHgEAAB8BAAAgAQAAIQEAACIBAAAjAQAAJAEAACUBAAAmAQAAJwEAACgBAAApAQAAKgEAACsBAAAsAQAALQEAAC4BAAAvAQAAMAEAADEBAAAyAQAAMwEAADQBAAA1AQAANgEAADcBAAA4AQAAOQEAADoBAAA7AQAAPAEAAD0BAAA+AQAAPwEAAEABAABBAQAAQgEAAEMBAABEAQAARQEAAEYBAABHAQAASAEAAEkBAABKAQAASwEAAEwBAABNAQAATgEAAE8BAABQAQAAUQEAAFIBAABTAQAAVAEAAFUBAABWAQAAVwEAAFgBAABZAQAAWgEAAFsBAABcAQAAXQEAAF4BAABfAQAAYAEAAGEBAABiAQAAYwEAAGQBAABlAQAAZgEAAGcBAABoAQAAaQEAAGoBAABrAQAAbAEAAG0BAABuAQAAbwEAAHABAABxAQAAcgEAAHMBAAB0AQAAdQEAAHYBAAB3AQAAeAEAAHkBAAB6AQAAewEAAHwBAAB9AQAAfgEAAH8BAAD+////gQEAAIIBAACDAQAAhAEAAIUBAACGAQAAhwEAAP7////9/////f////3////9////jQEAAP7////+/////v////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////9SAG8AbwB0ACAARQBuAHQAcgB5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFgAFAf//////////AwAAAAYJAgAAAAAAwAAAAAAAAEYAAAAAAAAAAAAAAAAgFZlgpnrRAY8BAACAAAAAAAAAADEAVABhAGIAbABlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOAAIB/////wUAAAD/////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAB4aAAAAAAAAVwBvAHIAZABEAG8AYwB1AG0AZQBuAHQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABoAAgEBAAAA//////////8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAFAFMAdQBtAG0AYQByAHkASQBuAGYAbwByAG0AYQB0AGkAbwBuAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAACAQIAAAAEAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABYAAAAM0wIAAAAAAAUARABvAGMAdQBtAGUAbgB0AFMAdQBtAG0AYQByAHkASQBuAGYAbwByAG0AYQB0AGkAbwBuAAAAAAAAAAAAAAA4AAIB////////////////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAEAAAAQAAAAAAAAAQBDAG8AbQBwAE8AYgBqAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABIAAgD///////////////8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAcgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP///////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////////////////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAP7///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8BAP7/AwoAAP////8GCQIAAAAAAMAAAAAAAABGIAAAAERvY3VtZW50IE1pY3Jvc29mdCBXb3JkIDk3LTIwMDQACgAAAE1TV29yZERvYwAQAAAAV29yZC5Eb2N1bWVudC44APQ5snEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" }
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - length: { _source.attachment: 6 }
+  - match: { _source.attachment.content: "Test elasticsearch" }
+  - match: { _source.attachment.language: "et" }
+  - match: { _source.attachment.author: "David Pilato" }
+  - match: { _source.attachment.date: "2016-03-10T08:25:00Z" }
+  - match: { _source.attachment.content_length: "19" }
+  - match: { _source.attachment.content_type: "application/msword" }
+
+
+---
+"Test ingest attachment processor with .docx file":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "attachment" : {
+                  "source_field" : "field1"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: { field1: "UEsDBBQABgAIAAAAIQBtiidLZgEAAFQFAAATAAgCW0NvbnRlbnRfVHlwZXNdLnhtbCCiBAIooAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC0lMtugzAQRfeV+g/I2wqcdFFVVUgWfSzbSE0/wLEH4tYv2c7r7ztAgqooAalJNkgwc+89A3hGk41WyQp8kNbkZJgNSAKGWyFNmZOv2Vv6SJIQmRFMWQM52UIgk/HtzWi2dRASVJuQk0WM7onSwBegWcisA4OVwnrNIt76kjrGf1gJ9H4weKDcmggmprHyIOPRCxRsqWLyusHHDQnKSfLc9FVROWHOKclZxDKtqvSozoMKHcKVEQd06Y4sQ2XdExbShbvTCd8OyoMEqavR6gJqPvB1eikgmTIf35nGBrq2XlBh+VKjKOse7gijLQrJodVXbs5bDiHgd9IqayuaSbNnP8kR4lZBuDxF49sfDzGi4BoAO+dehDXMP69G8ce8F6TA3BmbK7g8RmvdCxHx1EJzHZ7NUdt0RWLn1FsXcAv4f4y9P66VOsWBHfgou/+6NhGtz54Pqk0gQBzJpvVOHP8CAAD//wMAUEsDBBQABgAIAAAAIQDHwie8/wAAAN8CAAALAAgCX3JlbHMvLnJlbHMgogQCKKAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAArJLNSgMxEIDvgu8Q5t7NtoqINNuLCL2JrA8wJtPd6OaHZKrt2xtF1IVlEexx/j6+SWa9ObhBvFLKNngFy6oGQV4HY32n4LG9W1yDyIze4BA8KThShk1zfrZ+oAG5DOXexiwKxWcFPXO8kTLrnhzmKkTypbILySGXMHUyon7BjuSqrq9k+s2AZsQUW6Mgbc0FiPYY6X9s6YjRIKPUIdEipjKd2JZdRIupI1Zggr4v6fzZURUyyGmhy78Lhd3OaroNeu/I85QXHZi8ITOvhDHOGS1PaTTu+JF5C8lI85Wes1md9sO437snj3aYeJfvWvUcqfsQkqOzbN4BAAD//wMAUEsDBBQABgAIAAAAIQATqj6H9gAAADEDAAAcAAgBd29yZC9fcmVscy9kb2N1bWVudC54bWwucmVscyCiBAEooAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKySy2rDMBBF94X+g5h9LTt9UELkbEoh29b9AEUeP6gsCc304b+vaEjr0GC68PJeMfeeQbPZfg5WvGOk3jsFRZaDQGd83btWwUv1eHUPgli7WlvvUMGIBNvy8mLzhFZzGqKuDyRSiiMFHXNYS0mmw0FT5gO69NL4OGhOMrYyaPOqW5SrPL+TcZoB5Umm2NUK4q6+BlGNAf+T7ZumN/jgzduAjs9UyA/cPyNzWo5SrI4tsoKJmaVEkOdBbpYEabzjSu8t/mL8WHMQt0tCcJqdAHzLg1nMMRRLMhCPFiefcdBz9atF6/9cw9E5IsiTQy+/AAAA//8DAFBLAwQUAAYACAAAACEA9WKOYGUCAAAOBwAAEQAAAHdvcmQvZG9jdW1lbnQueG1spFXfb9owEH6ftP8h8jtNwijQiFDR0qI+TKpK9zwZx0ksYp9lGyj763dOIGSbVtGSh9j367vv7mJncvsmq2DLjRWgUhJfRSTgikEmVJGSH6+PvTEJrKMqoxUonpI9t+R2+vXLZJdkwDaSKxcghLLJTrOUlM7pJAwtK7mk9koKZsBC7q4YyBDyXDAe7sBkYT+Ko3qnDTBuLea7p2pLLTnASTgPTVJ23PajaIyyUC3Gv4xAc4XGHIykDkVTYIRZb3QPMTV1YiUq4fYea9jCbFOyMSo5YPRaHj4mQQLJVlZHZ3jPtyF6WI4R5hySTcj80PKaXmh4hYRB2VLoU98+i4bG8gjybsGdYnc6Hlw29LmhO1xOgOfQz5ogWTXM30eMozMm4iHaiHMo/JnzyKT78e0+15pOc+PrjwH0/wbQxWXDWRjY6BOauAztSa1bLH+VfADrMORuafYyMsuSajyBkiVPhQJDVxUywpEF2PXAf9ZkilfcCrK9XzWqB4mmhj5lKRmNhg/X9/GI1FrH31yjbR7UJnidZi8piaK7m8Hw5rpVzXlON5XzlvEwGs8f6yzGv9z0lVsX4JG2TjDLqWHlJPR6/65dVgBrf1ktHTUOIQVmjTy2ohLZ/1zAHWVrEnZ9H1TWeoY1lPZmy5l7Nv9nukS7185m8WjW9EIXy19oxdMRxzdRnbfE/XA8qJG9w3fqIR3gIY4HdX8SI4rSncQVOAfyJFc871hLTjOO1+EoGnsxB3Adsdi4WjykY1BZ1FpNGW98ajX+lRZG+KIrofizcAxZfhseq28Kr7fNcMPTj2z6GwAA//8DAFBLAwQUAAYACAAAACEAbU1ZqyEGAACOGgAAFQAAAHdvcmQvdGhlbWUvdGhlbWUxLnhtbOxZy47bNhTdF+g/ENo7lm3Jj0E8gS3bSZuZJMg4abOkJVpihhINkpoZIwjQLyhQIC26KdBdC3QToP2D/kuKNv2IUpRlkzbdQToOEBSxAYuPcy8P7yUPJev2nauUgAvEOKZZ32ncch2AspBGOIv7zpPppNZ1ABcwiyChGeo7S8SdO8effnIbHokEpQhI+4wfwb6TCLE4qtd5KJshv0UXKJN9c8pSKGSVxfWIwUvpNyX1puu26ynEmQMymEq30+T3n6Wzh/M5DpFzXHkfE/mTCV40hISdFb7RymSYMwRzhY3OG8WFL3lAGLiApO/IgSJ6OUVXwgEEciE7+o6rPk79+HZ9bUTEHlvNbqI+K7uVQXTeVHYsnq0NPc/32oO1fwUgYhc37ozb4/banwLAMJQzLbnoWH/YG478FVYDlUWL71Fn1GoYeM1/awc/8IuvgVegsujt4CeTYBNDDVQWfUtMOs3AM/AKVBbbO/iOOxh5HQOvQAnB2fkO2vXbraCa7Royp+SeFd7zvUmnuYJvUHVtdZX2mdi31lL4nLKJBKjkQoEzIJYLNIehxAWQ4BnD4ATHiVx4C5hRLpvdpjtxW/K3+HqqpCICjxDUrMumkO80FXwADxleiL7zufTqaJBnObhLRYLD1ai7FvdgFusWb3/65u8fvgJ//frj21ff2vFcx49QFn+JYfZvAwjd4M13r//47fWb77/+85dXFviAwZkOn+IUcfAAXYLHNJWTswyAZuzdLKYJxLrFIIs5zGBhY0GPZfx09IMlJNCCGyIzkk+ZlAob8G7+3CB8lrBcYAvwfpIawFNKyZAy65zuF2PpUciz2D44y3XcYwgvbGMHW3ke5wu55rHNZZAgg+YjIlMOY5QhAYo+eo6QxewZxkZcT3HIKKdzAZ5hMITYGpIpnhmraWN0D6cyL0sbQZlvIzanT8GQEpv7EbowkXJ3QGJziYgRxrswFzC1MoYp0ZEnUCQ2kmdLFhoB50JmOkaEgnGEOLfZPGRLg+59KTH2tJ+SZWoimcDnNuQJpNTY4PQ8SGC6sHLGWaJjP+PncolC8IgKKwlq7pCiLvMgxWNfup9iZKT7+r39RMqQfYEUPTmzbQlEzf24JHOIlPP6lqanOLtW4Lek3X9/0n6KszChds09iKjboTeR8wHD1v20LeL7cNvSHVAW4Q9fuUcwzx4huVks0I/C/VG4//fCvW8/H16uNwqtbuOrm3XlJt175z7HhJyJJUEnXGk7l9OLJrJRVZTR+kFhkcjiajgDFzOoyoBR8QUWyVkCF3KYhhoh5ivXMQcLyuXpoJqtvosOkqenNCpbG43q2VQaQLFpl6dL1S7PIlG2tjubh7C1e1WL1cNyRaCwfRcS2mAmiZaFRKdqvIaEmtlBWPQsLLqF+70s1GWVFbn/ACz+1/C9kpFcb5CgqMhTaV9l9+CZ3hdMc9pNy/R6BdfDZNogoS03k4S2DBMYoe3mA+e6t0mpQa8IxS6NTvd95LoQkS1tIJlZA5dyz7V86SaEi74zl/eFspgupD9e6CYkcdZ3QrEK9H9RlgXjYgR5UsJUVzn/FAvEAMGpXOt6Gki24dZodoo5fqDkeu6HFzl10ZOM5nMUij0tm6rsK51Ye28ILio0l6TPkugSzEjOHkMZKL/TKAIYYS7W0Yww0xb3JopbcrXaisZ/ZpstCskigasTRRfzEq7KazraPBTT7VmZ9dVkZnGRpBufutcbFR2aaO45QIpT064f7++Q11htdN9gVUr3ttb1Kq3bd0rc/EDQqG0GM6gVjC3UNq0mtQPeEGjDrZfmvjPi0KfB9qotDojqvlLVdl5O0NlzufJH8nY1J4IrquhKPiME1d/KpRKo1kpdrgTIGe47L1x/4AVNP6i5XX9c81qeW+v6g1Zt4PutxthvuKNh86UMikjShl+OPZHPM2S5evmi2ndewKTVbfatkKZ1qt6s1JWxegHTaBovYMo3L2Ba9DsAy8i8aDcnvVZv2K71WoNJzRsNu7Ve0B7WRu2gM5qMAr/bm7x0wIUCe4NW4LXH3Vq7EQQ1r+0W9Lu9WsdrNgdeZ9Ade4OXq1jLmVfXKryK1/E/AAAA//8DAFBLAwQKAAAAAAAAACEAvOgH/fQnAAD0JwAAFwAAAGRvY1Byb3BzL3RodW1ibmFpbC5qcGVn/9j/4AAQSkZJRgABAQAASABIAAD/4QCARXhpZgAATU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUAAAABAAAARgEoAAMAAAABAAIAAIdpAAQAAAABAAAATgAAAAAAAABIAAAAAQAAAEgAAAABAAOgAQADAAAAAQABAACgAgAEAAAAAQAAAWmgAwAEAAAAAQAAAgAAAAAA/+0AOFBob3Rvc2hvcCAzLjAAOEJJTQQEAAAAAAAAOEJJTQQlAAAAAAAQ1B2M2Y8AsgTpgAmY7PhCfv/AABEIAgABaQMBEQACEQEDEQH/xAAfAAABBQEBAQEBAQAAAAAAAAAAAQIDBAUGBwgJCgv/xAC1EAACAQMDAgQDBQUEBAAAAX0BAgMABBEFEiExQQYTUWEHInEUMoGRoQgjQrHBFVLR8CQzYnKCCQoWFxgZGiUmJygpKjQ1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4eLj5OXm5+jp6vHy8/T19vf4+fr/xAAfAQADAQEBAQEBAQEBAAAAAAAAAQIDBAUGBwgJCgv/xAC1EQACAQIEBAMEBwUEBAABAncAAQIDEQQFITEGEkFRB2FxEyIygQgUQpGhscEJIzNS8BVictEKFiQ04SXxFxgZGiYnKCkqNTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqCg4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2dri4+Tl5ufo6ery8/T19vf4+fr/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/3QAEAC7/2gAMAwEAAhEDEQA/AP7Yfgx8GPg9N8HvhRLL8KPhrLLL8NfAskkkngTws8kkj+F9LZ3d200s7uxLMzHczEk5JNAHpX/ClPg3/wBEl+GX/hBeFf8A5W0AH/ClPg3/ANEl+GX/AIQXhX/5W0AH/ClPg3/0SX4Zf+EF4V/+VtAB/wAKU+Df/RJfhl/4QXhX/wCVtAB/wpT4N/8ARJfhl/4QXhX/AOVtAB/wpT4N/wDRJfhl/wCEF4V/+VtAB/wpT4N/9El+GX/hBeFf/lbQAf8AClPg3/0SX4Zf+EF4V/8AlbQAf8KU+Df/AESX4Zf+EF4V/wDlbQAf8KU+Df8A0SX4Zf8AhBeFf/lbQAf8KU+Df/RJfhl/4QXhX/5W0AH/AApT4N/9El+GX/hBeFf/AJW0AH/ClPg3/wBEl+GX/hBeFf8A5W0AH/ClPg3/ANEl+GX/AIQXhX/5W0AH/ClPg3/0SX4Zf+EF4V/+VtAB/wAKU+Df/RJfhl/4QXhX/wCVtAB/wpT4N/8ARJfhl/4QXhX/AOVtAB/wpT4N/wDRJfhl/wCEF4V/+VtAB/wpT4N/9El+GX/hBeFf/lbQAf8AClPg3/0SX4Zf+EF4V/8AlbQAf8KU+Df/AESX4Zf+EF4V/wDlbQAf8KU+Df8A0SX4Zf8AhBeFf/lbQAf8KU+Df/RJfhl/4QXhX/5W0AH/AApT4N/9El+GX/hBeFf/AJW0AH/ClPg3/wBEl+GX/hBeFf8A5W0AH/ClPg3/ANEl+GX/AIQXhX/5W0AH/ClPg3/0SX4Zf+EF4V/+VtAB/wAKU+Df/RJfhl/4QXhX/wCVtAB/wpT4N/8ARJfhl/4QXhX/AOVtAB/wpT4N/wDRJfhl/wCEF4V/+VtAB/wpT4N/9El+GX/hBeFf/lbQAf8AClPg3/0SX4Zf+EF4V/8AlbQAf8KU+Df/AESX4Zf+EF4V/wDlbQAf8KU+Df8A0SX4Zf8AhBeFf/lbQAf8KU+Df/RJfhl/4QXhX/5W0AH/AApT4N/9El+GX/hBeFf/AJW0AH/ClPg3/wBEl+GX/hBeFf8A5W0AH/ClPg3/ANEl+GX/AIQXhX/5W0AH/ClPg3/0SX4Zf+EF4V/+VtAB/wAKU+Df/RJfhl/4QXhX/wCVtAB/wpT4N/8ARJfhl/4QXhX/AOVtAB/wpT4N/wDRJfhl/wCEF4V/+VtAB/wpT4N/9El+GX/hBeFf/lbQAf8AClPg3/0SX4Zf+EF4V/8AlbQB/Nd/wrT4c/8ARP8AwT/4Sug//INAH//Q/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9H+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/T/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9T+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAeDfEX4/+Hvhf8QfD3gbxN4W8Vx6ZrfgHxz8TNQ+JS33w9sPh34O8G/DOXRk8fav4uvfEHj3RPE1jD4Xt/Evhu/v307wrq0M1hrUU+ny3Z03Xk0oAxf8Ahrr9n+W68P2WneN7nXLrxJ4x0D4f2sHh7wj411x9L8aeI9T13RrPwx4qGmeHrr/hD9esNV8Na1YeJNG8Uto+q+Eriz2+KLLRxNbvKAXNd/am+C2h+MbHwIPFDa34kn8V6n4Q1Ox8OWU+sSaBf6R8P/iz8Q7+7v4YALrVNPt7T4K+PPCs58JW/ia/tfH2nf8ACHXun22rW2qRaWAZN7+2L+z3bRaLNY+N5/ECa/4p0XwRYyeG/DPinWLW38XeIPh9r/xR07w7rupW+jHSvC2qQ+B/Dt5r2vweJ77SB4Ns7zRZvGjeHoNc0qa6ANXSP2r/ANn7WUia2+JWhxg6X4r1q7mlNxLpWl6R4Asbe7+IWsaj4lsobvwrD4f+H91dQeHvGvimLXZ/C3hzxhJH4O1LWovE8sWkOANvf2s/2eNN06x1bUviZpmnadqdvey6fdahpXiSyS7v9N8cH4a6l4YgW50aGR/HenePlfwnf/D0L/wnNlrUctnc+HonikKgGaP2wfgC/wARfCHwzh8arNrHje68eaVoWsLYXkXhaTxJ8O/Gfw8+Hmu+GJtcuY4Ihqs/jj4m+HPB+l3MME+g3njFbzwS+tW/jP8As/QL8A+nKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoA/mXoA//1f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQB5b8Sfgr8MPi/bahZ/EfwrB4otNV+HPxJ+Emo2t1qGsWltefDz4vW2gWfxF8OTw6ZqNlG0XiO18MaJDJf7Rq2mrZE6Pf6c11etcAHk2k/sUfs26Fq/hbXtJ8DavYaz4JstA07wrqNt8R/ifHc6LY+G/Gcfj/TbS0ZfGKqLZ/FKTX2p28ivBrFpqGs6LqsV3omu65p2oAEGtfsOfsueILvx9fal8MAbj4neIfEXivxo1j4y8f6Qmp+IvF3hL4ieCPE+rW0Ok+KLKDRbjXfDvxY+IUOprocenQXWpeJbrxBJEfEFtp+p2gBQ0H9gr9lfwv8AYz4f+HWraW2nW/hSx06S2+J3xZ86w0/wXbeLLLQ9MtLiTx01xBpf2Dx54107WdMSQWHiTT/FWvWXiKDVLbVLtJQB17+wd+y1qWhWfhXU/h9rOqeEdPvfH9/p3g7VPij8W9S8H6dc/FLRPEnh/wCIT6f4WvvHU2g2K+LNM8Y+K01aO10+FJbrxFrGoIqX17PcMAV9M/YB/ZM0jxR4H8Z2PwtuF8S/Di38NWvgzUZ/iF8T7saNF4R+Is3xY0DFjc+M5tNv5LP4gTvr8s2p2l5JfYTS79rnRkTT6AO9sf2UfgVp97a30PhPWJ5dP8S3vizR4NR+IPxI1XTvD+rah8V/AHxwuYvDek6n4vvNL8OaGPin8MPBPiu28LaJZ2Hhmxk0iXSbHSbfQNX1rS9SAPoqgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKAP5l6AP/W/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9f+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/R/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9L+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/U/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9X+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/X/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9D+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/S/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9P+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/V/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9b+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/Q/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9H+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/T/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9T+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/W/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9f+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/R/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9L+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/U/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9X+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/X/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9D+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/S/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9P+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/V/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9b+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/Q/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9H+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/T/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9T+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/W/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9f+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/R/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9L+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/U/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9X+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1v7Yfgx8Z/g9D8HvhRFL8V/hrFLF8NfAsckcnjvwskkcieF9LV0dG1IMjowKsrDcrAg4INAHpX/C6/g3/wBFa+GX/he+Ff8A5ZUAH/C6/g3/ANFa+GX/AIXvhX/5ZUAH/C6/g3/0Vr4Zf+F74V/+WVAB/wALr+Df/RWvhl/4XvhX/wCWVAB/wuv4N/8ARWvhl/4XvhX/AOWVAB/wuv4N/wDRWvhl/wCF74V/+WVAB/wuv4N/9Fa+GX/he+Ff/llQAf8AC6/g3/0Vr4Zf+F74V/8AllQAf8Lr+Df/AEVr4Zf+F74V/wDllQAf8Lr+Df8A0Vr4Zf8Ahe+Ff/llQAf8Lr+Df/RWvhl/4XvhX/5ZUAH/AAuv4N/9Fa+GX/he+Ff/AJZUAH/C6/g3/wBFa+GX/he+Ff8A5ZUAH/C6/g3/ANFa+GX/AIXvhX/5ZUAH/C6/g3/0Vr4Zf+F74V/+WVAB/wALr+Df/RWvhl/4XvhX/wCWVAB/wuv4N/8ARWvhl/4XvhX/AOWVAB/wuv4N/wDRWvhl/wCF74V/+WVAB/wuv4N/9Fa+GX/he+Ff/llQAf8AC6/g3/0Vr4Zf+F74V/8AllQAf8Lr+Df/AEVr4Zf+F74V/wDllQAf8Lr+Df8A0Vr4Zf8Ahe+Ff/llQAf8Lr+Df/RWvhl/4XvhX/5ZUAH/AAuv4N/9Fa+GX/he+Ff/AJZUAH/C6/g3/wBFa+GX/he+Ff8A5ZUAH/C6/g3/ANFa+GX/AIXvhX/5ZUAH/C6/g3/0Vr4Zf+F74V/+WVAB/wALr+Df/RWvhl/4XvhX/wCWVAB/wuv4N/8ARWvhl/4XvhX/AOWVAB/wuv4N/wDRWvhl/wCF74V/+WVAB/wuv4N/9Fa+GX/he+Ff/llQAf8AC6/g3/0Vr4Zf+F74V/8AllQAf8Lr+Df/AEVr4Zf+F74V/wDllQAf8Lr+Df8A0Vr4Zf8Ahe+Ff/llQAf8Lr+Df/RWvhl/4XvhX/5ZUAH/AAuv4N/9Fa+GX/he+Ff/AJZUAH/C6/g3/wBFa+GX/he+Ff8A5ZUAH/C6/g3/ANFa+GX/AIXvhX/5ZUAH/C6/g3/0Vr4Zf+F74V/+WVAB/wALr+Df/RWvhl/4XvhX/wCWVAB/wuv4N/8ARWvhl/4XvhX/AOWVAB/wuv4N/wDRWvhl/wCF74V/+WVAB/wuv4N/9Fa+GX/he+Ff/llQAf8AC6/g3/0Vr4Zf+F74V/8AllQB/Nd/wsv4c/8ARQPBP/hVaD/8nUAf/9kAAFBLAwQUAAYACAAAACEAuN5y8JsDAACACQAAEQAAAHdvcmQvc2V0dGluZ3MueG1stFZLj9s2EL4X6H8wdK5Wj8iOV403sL1xs8E6WazcS2+URNnE8iEMKatO0f/eESWunGYRuA3ii8n55s1vxn7z9k/BJ0cKmim58KKr0JtQWaiSyf3C+3238efeRBsiS8KVpAvvRLX39ubnn960qabGoJqeoAupU1EsvIMxdRoEujhQQfSVqqlEsFIgiMEr7ANB4Kmp/UKJmhiWM87MKYjDcOYNbtTCa0CmgwtfsAKUVpXpTFJVVaygw5ezgEvi9ia3qmgElcZGDIByzEFJfWC1dt7E//WG4ME5OX6riKPgTq+NwgvKbRWUzxaXpNcZ1KAKqjU+kOAuQSbHwMlXjp5jX2HsoUTrCs2j0J7OM5/+NwfxvxxofkklPXTPciDQ82QoQxTp3V4qIDlHVmI5E8zIu0FaflZKTNq0plDg2yCnw9ALOgA7oqrMEEMR1jXl3JK84JSgwzbdAxFITyexNiWtSMPNjuSZUTUqHQnm/Tqc9/DhVB+otCT6A8fD4Uk87fHiQIAUhkJWkwKjrZU0oLjTK9VHZdY4CoAv1VtocqQPQI+Mtg+sMA3Q3pGdl/GU9bOHjiQR2IAv5mmrStoV1AC7/I06A5tU5HJ/MZDCXQGspLuu8Zk5cbrBmjL2mS5l+aHRhqFH25DvyOBbCWC7MfInpMruVNMNJV2P9A8KZh9ow1m9ZQAK7mSJlPphwVhVUcAADCm6RdYxUK3t83tKSlzR3xk3OKcRLvxSu8OjUsaphuF8Fs5vN32mHXoJslxGr5fJS8jqOpldW0oFz1FF2i3LB3CnjkIT0VusiciBkcm2W6dBp5HD04pJh+cU9wM9R7Imd6Dv94AWhPMNjp4D7AoQacl0fUsre+ZbAvvR76ABL0pxDXx49tWtFQq/gWrqHm2B1D01nEqUJIMlk+aeCSfXTZ45K4kb7QxqZPnpCLZPY3va1OAT2xG7J5YqVrcCf/M4UIlD1tGAbkld92zK99HC42x/MFFHAIO3En917SXfxwMWWyzuMXshRVcZag+HURY72ZneKyd7NcoSJ0tG2dTJpqNs5mSzToZLlALu4icktjt28kpxrlpavh/xr0RuSxcMXzw7iXxcrr/0GGcaJ63GPWwUOOxXi0VJWqriDsmKp/655+t3yTxa9vDU7m+zQx49YWsfabUimpYD5kynvelfm+4zj1f+MrqN/WQ2XfnzeP3OX23iZbReXs+m6/jvYQ7cX6ebfwAAAP//AwBQSwMEFAAGAAgAAAAhAPC8NQHcAQAA8QUAABIAAAB3b3JkL2ZvbnRUYWJsZS54bWy8k9tq4zAQhu8LfQej+8ay4vRg6pQ0bWBh6cXSfQBFkW2xOhhJiTdvvyPZcQMhbJallUHI/4x+jT40j0+/lUx23DphdImyCUYJ18xshK5L9PN9dXOPEuep3lBpNC/Rnjv0NL++euyKymjvEtivXaFYiRrv2yJNHWu4om5iWq4hWBmrqIdfW6eK2l/b9oYZ1VIv1kIKv08JxrdosLGXuJiqEoy/GLZVXPu4P7VcgqPRrhGtO7h1l7h1xm5aaxh3Du6sZO+nqNCjTZafGCnBrHGm8hO4zFBRtILtGY4rJT8MZv9mQEYDxYpvtTaWriXAh0oSMEPzgX7SFZoqCCypFGsrYqCl2jieQWxHZYkwwSs8gzl8OZ6GGaUhkTXUOh5M+kTcyxVVQu4PKt160+ut8Kw5yDtqRaipDzlRQ2Dr1rhErxgGWa1Qr2QlykFYLEeFhKPiyAZlOio4KCz69BkPcReLPmMOnJn2AE5AvAvFXfLGu+SHUVSfAULwLYCYAY4AZvr5QMji9QjIEpS7+/xw/Q8gD38H0mO8HMgCypJnMDwDhnx4GfF1fD6G43cxYJh+BYahQZLvom782TYJzfFFbbIIFZPjVxHahOC75xMc8fL/2SbDws3/AAAA//8DAFBLAwQUAAYACAAAACEA4IvKVR8BAAARAgAAFAAAAHdvcmQvd2ViU2V0dGluZ3MueG1slNFRS8MwEAfwd8HvUPK+pRs6tKwbgkz2MgbVD5Cl1zWY5EIua7dv71nnRHyZbzku9+P+3Hx5dDbrIJJBX4rJOBcZeI218ftSvL2uRg8io6R8rSx6KMUJSCwXtzfzvuhhV0FK/JMyVjwVTpeiTSkUUpJuwSkaYwDPzQajU4nLuJdOxfdDGGl0QSWzM9akk5zm+UycmXiNgk1jNDyjPjjwaZiXESyL6Kk1gb61/hqtx1iHiBqIOI+zX55Txl+Yyd0fyBkdkbBJYw5z3migeHySDy9nf4D7/wHTC+B0sd57jGpn+QS8ScaYWPANlLXYbzcv8rOocYOpUh08UcUpLKyMhaETzBEsbSGuvW6zvuiULcXjTHBT/jrk4gMAAP//AwBQSwMEFAAGAAgAAAAhABZNBGBtAQAA7wIAABEACAFkb2NQcm9wcy9jb3JlLnhtbCCiBAEooAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJySUW+CMBSF35fsP5C+Q4suxhDAZDM+zcRkLlv21rVX7YS2aavIv18BxbH5tLd7e757uJw2nZ3KIjiCsULJDMURQQFIpriQ2wy9rhfhFAXWUclpoSRkqAaLZvn9Xcp0wpSBlVEajBNgA+8kbcJ0hnbO6QRjy3ZQUht5Qnpxo0xJnW/NFmvK9nQLeETIBJfgKKeO4sYw1L0jOlty1lvqgylaA84wFFCCdBbHUYyvrANT2psDrfKDLIWrNdxEL2JPn6zowaqqomrcon7/GL8vn1/aXw2FbLJigPKUs8QJV0Ce4mvpK3v4/ALmuuO+8TUzQJ0y+ZweBQ9WovBdC12EJvI91JUy3PrxQecxDpYZoZ2/yM58cODpglq39De7EcAf61/f+as3IwaOonkZedwSfZueY+52Ax74eJIuzIvyNn6arxcoH5F4EpJxGJM1mSajh4SQj2a9wfzVsDwv8G/Hi0GX0PCJ5t8AAAD//wMAUEsDBBQABgAIAAAAIQCBlv05MgsAAGRyAAAPAAAAd29yZC9zdHlsZXMueG1svJ3bctu6FYbvO9N34OiqvXB8jJ14trPHduLaUzvbO3Kaa4iEJNQgofLgQ5++IEhJkBdBcQGrvrIlan0A8eMHsEBS+u33l1RGTzwvhMrORvsf9kYRz2KViGx2Nvr5cLXzaRQVJcsSJlXGz0avvBj9/uWvf/nt+bQoXyUvIg3IitM0PhvNy3JxurtbxHOesuKDWvBMH5yqPGWlfpnPdlOWP1aLnVilC1aKiZCifN092Ns7HrWYfAhFTaci5l9VXKU8K038bs6lJqqsmItFsaQ9D6E9qzxZ5CrmRaFPOpUNL2UiW2H2jwAoFXGuCjUtP+iTaWtkUDp8f8/8l8o14CMOcLACpPHpzSxTOZtI3fq6JpGGjb7o5k9U/JVPWSXLon6Z3+fty/aV+XOlsrKInk9ZEQvxoEvWkFRo3vV5VoiRPsJZUZ4XgnUenNf/dB6Ji9J6+0IkYrRbl1j8Vx98YvJsdHC0fOeyrsHGe5Jls+V703zn6oddk7MRz3Z+juu3Jpp7NmL5zvi8DtxtT6z5a53uYvWq+dSbttFdQ3eUcdNf9VE+vVXxI0/GpT5wNtqri9Jv/ry5z4XKdZ88G33+3L455qm4FknCM+uD2Vwk/NecZz8Lnqzf//PK9Kv2jVhVmf7/8NOe0UsWybeXmC/qXqqPZqxuve91gKw/XYl14Sb8P0vYfttmXfFzzmqrRvtvEab6KMRBHVFYZ9vNrN6cu/kUqqDD9yro6L0K+vheBR2/V0En71XQp/cqyGD+nwWJLOEvjRFhMYC6jeNwI5rjMBua4/ASmuOwCprjcAKa4+joaI6jH6M5jm6K4JQqdvVCq7MfOnp7P3f7HOHH3T4l+HG3zwB+3O0Dvh93+/jux90+nPtxt4/eftztgzWe2yy1ohtts6wMdtlUqTJTJY9K/hJOY5lmmfyFhldPejwnOUkCTDOytRNxMC1m5vX2HmJM6j+fl3XKFalpNBWzKtdpb2jFefbEpU5AI5YkmkcIzHlZ5Y4W8enTOZ/ynGcxp+zYdFApMh5lVToh6JsLNiNj8Swhbr4lkWRQWHVoVpXz2iSCoFOnLM5VeNUUIxsfbkUR3lY1JLqopORErO80XcywwnMDgwlPDQwmPDMwmPDEwNKMqolaGlFLtTSiBmtpRO3W9E+qdmtpRO3W0ojaraWFt9uDKKUZ4u1Vx/7wvbtLqeod5+B6jMUsY3oBED7dtHum0T3L2Sxni3lU7x93Y+1zxpZzoZLX6IFiTluRqNb1potc6rMWWRXeoBs0KnOteET2WvGIDLbihVvsTi+T6wXaNU0+M64mZadpDWmQacdMVs2CNtxtrAzvYWsDXIm8ILNBN5agB3+vl7O1nBQj37qW4RVbs8Jt9XZUIq1eiySopVTxI80wfP264LlOyx6DSVdKSvXMEzriuMxV09dsyx8YSQZZ/lu6mLNCmFxpAzF8ql9eq47u2CL4hO4lExmNbt92UiZkRLeCuH64u40e1KJOM+uGoQFeqLJUKRmz3Qn82y8++TtNBc91Epy9Ep3tOdH2kIFdCoJJpiGphIikl5kiEyRzqOH9k79OFMsTGtp9zpvbQ0pORByzdNEsOgi8pcfFZz3+EKyGDO9fLBf1vhCVqR5IYNa2YVFN/s3j8KHuu4pIdob+qEqz/2iWuiaaDhe+TNjAhS8RjJp6eqj7L8HJbuDCT3YDR3Wyl5IVhXBeQvXmUZ3ukkd9vuHJX8tTUuXTStI14BJI1oJLIFkTKlmlWUF5xoZHeMKGR32+hF3G8Ai25AzvH7lIyMQwMColDIxKBgOj0sDASAUIv0PHgoXfpmPBwu/VaWBESwALRtXPSKd/oqs8FoyqnxkYVT8zMKp+ZmBU/ezwa8SnU70IpptiLCRVn7OQdBNNVvJ0oXKWvxIhv0k+YwQbpA3tPlfT+rkBlTU3cRMg6z1qSbjYbnBUIv/iE7Kq1SzKehHsiDIplSLaW1tPOCZy8961bWHmmYvgKpjN9lv+xClW4xaM6DJAAwuXzYKFT1MWLHyasmDh05QFC5+mLFj4NGXBwu9fvpcs5nMlE547jNhXkWi8YHF7bQlcox60V38rZvMyGs9Xl6hszPHe1sjlLtNG2PYCuwaK44OesDueiCpdVhQ+AXR8ODzYGHojePmgVk/wevm7EflxYCQs83h75Dq124g8GRgJy/w0MNKMUhuRfYP4V5Y/dnaEk77+s9qYcHS+k75etAruLLavI60iu7rgSV8v2rBKdB7H9SUuqM4wz7jjh5nHHY9xkZuCsZObMthXbkSfwX7wJ1EvRzGDpilvdcvP2+IOzZQ6aOT8s1LNxaaNq6TDn0S80av9rOBRJ+dw+NXWjVHG3Y6Dhxs3YvC440YMHoDciEEjkTMcNSS5KYPHJjdi8CDlRqBHKzgj4EYrGI8brWC8z2gFKT6jVcAqwI0YvBxwI9BGhQi0UQNWCm4Eyqgg3MuokII2KkSgjQoRaKPCBRjOqDAeZ1QY72NUSPExKqSgjQoRaKNCBNqoEIE2KkSgjeq5tneGexkVUtBGhQi0USECbVSzXgwwKozHGRXG+xgVUnyMCiloo0IE2qgQgTYqRKCNChFoo0IEyqgg3MuokII2KkSgjQoRaKM2z8f6GxXG44wK432MCik+RoUUtFEhAm1UiEAbFSLQRoUItFEhAmVUEO5lVEhBGxUi0EaFCLRRzaWDAKPCeJxRYbyPUSHFx6iQgjYqRKCNChFoo0IE2qgQgTYqRKCMCsK9jAopaKNCBNqoENHXP9vr6q5nQ/bxu57Ox0yGX7pqK/XD/v4BG3U4HLWslZs1/AGaC6Ueo86nZQ9NvjEMIiZSKLNF7bgXxOaaC6Soq/V/XPY/lmbTA78prH2Ax1zoB/CjoZFgT+Wor8vbkSDJO+rr6XYkWHUe9Y2+diSYBo/6Bl3jy+WdVHo6AsF9w4wVvO8I7xutrXDYxH1jtBUIW7hvZLYCYQP3jcdW4MeoHpzfRn8c2E7Hq5uiAaGvO1qEEzehr1tCrZbDMTTGUNHchKHquQlDZXQTUHo6MXhh3Si0wm6Un9TQZlip/Y3qJmClhgQvqQHGX2qI8pYaovykhgMjVmpIwErtPzi7CV5SA4y/1BDlLTVE+UkNpzKs1JCAlRoSsFIHTshOjL/UEOUtNUT5SQ0Xd1ipIQErNSRgpYYEL6kBxl9qiPKWGqL8pAZZMlpqSMBKDQlYqSHBS2qA8Zcaorylhqg+qc0uyobUKIWtcNwizArETchWIG5wtgI9siUr2jNbsgie2RLUaqk5LluyRXMThqrnJgyV0U1A6enE4IV1o9AKu1F+UuOypS6p/Y3qJmClxmVLTqlx2VKv1LhsqVdqXLbklhqXLXVJjcuWuqT2H5zdBC+pcdlSr9S4bKlXaly25JYaly11SY3LlrqkxmVLXVIHTshOjL/UuGypV2pctuSWGpctdUmNy5a6pMZlS11S47Ilp9S4bKlXaly21Cs1LltyS43LlrqkxmVLXVLjsqUuqXHZklNqXLbUKzUuW+qV2pEt7T5v/GpYzTa/d6c/XL4ueP3F8dYDM0nzxbntRUDzwZtk9etedXBdk6j9xbP2bVPh9oJhU6IJhEXFc11W3H7ll6OoeyWFPm+WJ/pwCYp0fLOvqcL65JefbhtzfRG0+dzGBc/eGpd1Y/fU1ojBqt72aRRzVfFz2wW31VHXaCKbH8PT/9xkiQY8t7+w1tQ1eWENSh+/5FLesebTauH+qOTTsjm6v2cen31zfNJ8YaEzPjeDhBOwu1mZ5mX7w3eOFm9+wqC9eu1o9fMqrjIutRt4R5ub+ylCm3tdweV/xZf/AQAA//8DAFBLAwQUAAYACAAAACEAQP7QLGkBAAC3AgAAEAAIAWRvY1Byb3BzL2FwcC54bWwgogQBKKAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACcUk1LxTAQvAv+h9K7L32CH8i+iCjiQUV4Vc8h2bbBNAnJKr5/78ZqrXgzp92ZZHZmCZy/j656w5Rt8Jt6vWrqCr0Oxvp+Uz+21wendZVJeaNc8Lipd5jrc7m/Bw8pRExkMVcs4fOmHojimRBZDziqvGLaM9OFNCriNvUidJ3VeBX064iexGHTHAt8J/QGzUGcBetJ8eyN/itqgi7+8lO7i6wnocUxOkUo78tLtzKBRhAzCm0g5Vo7omwYnht4UD1muQYxFfAcksnyEMRUwOWgktLE+5PrExCLFi5idFYr4sXKO6tTyKGj6k5p6ynkoSoKIJa3gENsUb8mS7viY9nCrfWTk6lgZ0n1ScXhy97cwVYrh5ccX3bKZQTxAxSVl/wY23BVYn/xv8FFpmdLwzYqXQafLtMtCNgyioa9zuNmAG54/ckVeX7rezTfd/4SZV9P0z+U66NVw+dzO98YZ5w/iPwAAAD//wMAUEsBAi0AFAAGAAgAAAAhAG2KJ0tmAQAAVAUAABMAAAAAAAAAAAAAAAAAAAAAAFtDb250ZW50X1R5cGVzXS54bWxQSwECLQAUAAYACAAAACEAx8InvP8AAADfAgAACwAAAAAAAAAAAAAAAACfAwAAX3JlbHMvLnJlbHNQSwECLQAUAAYACAAAACEAE6o+h/YAAAAxAwAAHAAAAAAAAAAAAAAAAADPBgAAd29yZC9fcmVscy9kb2N1bWVudC54bWwucmVsc1BLAQItABQABgAIAAAAIQD1Yo5gZQIAAA4HAAARAAAAAAAAAAAAAAAAAAcJAAB3b3JkL2RvY3VtZW50LnhtbFBLAQItABQABgAIAAAAIQBtTVmrIQYAAI4aAAAVAAAAAAAAAAAAAAAAAJsLAAB3b3JkL3RoZW1lL3RoZW1lMS54bWxQSwECLQAKAAAAAAAAACEAvOgH/fQnAAD0JwAAFwAAAAAAAAAAAAAAAADvEQAAZG9jUHJvcHMvdGh1bWJuYWlsLmpwZWdQSwECLQAUAAYACAAAACEAuN5y8JsDAACACQAAEQAAAAAAAAAAAAAAAAAYOgAAd29yZC9zZXR0aW5ncy54bWxQSwECLQAUAAYACAAAACEA8Lw1AdwBAADxBQAAEgAAAAAAAAAAAAAAAADiPQAAd29yZC9mb250VGFibGUueG1sUEsBAi0AFAAGAAgAAAAhAOCLylUfAQAAEQIAABQAAAAAAAAAAAAAAAAA7j8AAHdvcmQvd2ViU2V0dGluZ3MueG1sUEsBAi0AFAAGAAgAAAAhABZNBGBtAQAA7wIAABEAAAAAAAAAAAAAAAAAP0EAAGRvY1Byb3BzL2NvcmUueG1sUEsBAi0AFAAGAAgAAAAhAIGW/TkyCwAAZHIAAA8AAAAAAAAAAAAAAAAA40MAAHdvcmQvc3R5bGVzLnhtbFBLAQItABQABgAIAAAAIQBA/tAsaQEAALcCAAAQAAAAAAAAAAAAAAAAAEJPAABkb2NQcm9wcy9hcHAueG1sUEsFBgAAAAAMAAwABgMAAOFRAAAAAA==" }
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - length: { _source.attachment: 6 }
+  - match: { _source.attachment.content: "Test elasticsearch" }
+  - match: { _source.attachment.language: "et" }
+  - match: { _source.attachment.author: "David Pilato" }
+  - match: { _source.attachment.date: "2016-03-10T08:24:00Z" }
+  - match: { _source.attachment.content_length: "19" }
+  - match: { _source.attachment.content_type: "application/vnd.openxmlformats-officedocument.wordprocessingml.document" }
+
diff --git a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yaml b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yaml
index b924484aa7d..cf86f4c7f4c 100644
--- a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yaml
+++ b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yaml
@@ -8,4 +8,18 @@
         nodes.info: {}
 
     - match:  { nodes.$master.plugins.0.name: ingest-geoip  }
-    - match:  { nodes.$master.ingest.processors.3.type: geoip }
+    - match:  { nodes.$master.ingest.processors.0.type: append }
+    - match:  { nodes.$master.ingest.processors.1.type: convert }
+    - match:  { nodes.$master.ingest.processors.2.type: date }
+    - match:  { nodes.$master.ingest.processors.3.type: fail }
+    - match:  { nodes.$master.ingest.processors.4.type: foreach }
+    - match:  { nodes.$master.ingest.processors.5.type: geoip }
+    - match:  { nodes.$master.ingest.processors.6.type: gsub }
+    - match:  { nodes.$master.ingest.processors.7.type: join }
+    - match:  { nodes.$master.ingest.processors.8.type: lowercase }
+    - match:  { nodes.$master.ingest.processors.9.type: remove }
+    - match:  { nodes.$master.ingest.processors.10.type: rename }
+    - match:  { nodes.$master.ingest.processors.11.type: set }
+    - match:  { nodes.$master.ingest.processors.12.type: split }
+    - match:  { nodes.$master.ingest.processors.13.type: trim }
+    - match:  { nodes.$master.ingest.processors.14.type: uppercase }
diff --git a/plugins/lang-javascript/build.gradle b/plugins/lang-javascript/build.gradle
index dae5204db20..1f431241838 100644
--- a/plugins/lang-javascript/build.gradle
+++ b/plugins/lang-javascript/build.gradle
@@ -28,7 +28,7 @@ dependencies {
 
 integTest {
   cluster {
-    systemProperty 'es.script.inline', 'true'
-    systemProperty 'es.script.indexed', 'true'
+    setting 'script.inline', 'true'
+    setting 'script.indexed', 'true'
   }
 }
diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/SimpleBench.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/SimpleBench.java
deleted file mode 100644
index 3445c116057..00000000000
--- a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/SimpleBench.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.script.javascript;
-
-import org.elasticsearch.common.StopWatch;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.script.CompiledScript;
-import org.elasticsearch.script.ExecutableScript;
-import org.elasticsearch.script.ScriptService;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- *
- */
-public class SimpleBench {
-
-    public static void main(String[] args) {
-        JavaScriptScriptEngineService se = new JavaScriptScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
-        Object compiled = se.compile("x + y", Collections.emptyMap());
-        CompiledScript compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, "testExecutableNoRuntimeParams", "js", compiled);
-
-        Map<String, Object> vars = new HashMap<String, Object>();
-        // warm up
-        for (int i = 0; i < 1000; i++) {
-            vars.put("x", i);
-            vars.put("y", i + 1);
-            se.executable(compiledScript, vars).run();
-        }
-
-        final long ITER = 100000;
-
-        StopWatch stopWatch = new StopWatch().start();
-        for (long i = 0; i < ITER; i++) {
-            se.executable(compiledScript, vars).run();
-        }
-        System.out.println("Execute Took: " + stopWatch.stop().lastTaskTime());
-
-        stopWatch = new StopWatch().start();
-        ExecutableScript executableScript = se.executable(compiledScript, vars);
-        for (long i = 0; i < ITER; i++) {
-            executableScript.run();
-        }
-        System.out.println("Executable Took: " + stopWatch.stop().lastTaskTime());
-
-        stopWatch = new StopWatch().start();
-        executableScript = se.executable(compiledScript, vars);
-        for (long i = 0; i < ITER; i++) {
-            for (Map.Entry<String, Object> entry : vars.entrySet()) {
-                executableScript.setNextVar(entry.getKey(), entry.getValue());
-            }
-            executableScript.run();
-        }
-        System.out.println("Executable (vars) Took: " + stopWatch.stop().lastTaskTime());
-    }
-}
diff --git a/plugins/lang-python/build.gradle b/plugins/lang-python/build.gradle
index 0980d7f62c9..c7466316806 100644
--- a/plugins/lang-python/build.gradle
+++ b/plugins/lang-python/build.gradle
@@ -28,8 +28,8 @@ dependencies {
 
 integTest {
   cluster {
-    systemProperty 'es.script.inline', 'true'
-    systemProperty 'es.script.indexed', 'true'
+    setting 'script.inline', 'true'
+    setting 'script.indexed', 'true'
   }
 }
 
diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/SimpleBench.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/SimpleBench.java
deleted file mode 100644
index d9559aef16c..00000000000
--- a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/SimpleBench.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.script.python;
-
-import org.elasticsearch.common.StopWatch;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.script.CompiledScript;
-import org.elasticsearch.script.ExecutableScript;
-import org.elasticsearch.script.ScriptService;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- *
- */
-public class SimpleBench {
-
-    public static void main(String[] args) {
-        PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
-        Object compiled = se.compile("x + y", Collections.emptyMap());
-        CompiledScript compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, "SimpleBench", "python", compiled);
-
-
-        Map<String, Object> vars = new HashMap<String, Object>();
-        // warm up
-        for (int i = 0; i < 1000; i++) {
-            vars.put("x", i);
-            vars.put("y", i + 1);
-            se.executable(compiledScript, vars).run();
-        }
-
-        final long ITER = 100000;
-
-        StopWatch stopWatch = new StopWatch().start();
-        for (long i = 0; i < ITER; i++) {
-            se.executable(compiledScript, vars).run();
-        }
-        System.out.println("Execute Took: " + stopWatch.stop().lastTaskTime());
-
-        stopWatch = new StopWatch().start();
-        ExecutableScript executableScript = se.executable(compiledScript, vars);
-        for (long i = 0; i < ITER; i++) {
-            executableScript.run();
-        }
-        System.out.println("Executable Took: " + stopWatch.stop().lastTaskTime());
-
-        stopWatch = new StopWatch().start();
-        executableScript = se.executable(compiledScript, vars);
-        for (long i = 0; i < ITER; i++) {
-            for (Map.Entry<String, Object> entry : vars.entrySet()) {
-                executableScript.setNextVar(entry.getKey(), entry.getValue());
-            }
-            executableScript.run();
-        }
-        System.out.println("Executable (vars) Took: " + stopWatch.stop().lastTaskTime());
-    }
-}
diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
index ab50d44a5c7..cf5a0cf41d7 100644
--- a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
+++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
@@ -28,6 +28,7 @@ import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -71,9 +72,12 @@ import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField;
 public class AttachmentMapper extends FieldMapper {
 
     private static ESLogger logger = ESLoggerFactory.getLogger("mapper.attachment");
-    public static final Setting<Boolean> INDEX_ATTACHMENT_IGNORE_ERRORS_SETTING = Setting.boolSetting("index.mapping.attachment.ignore_errors", true, false, Setting.Scope.INDEX);
-    public static final Setting<Boolean> INDEX_ATTACHMENT_DETECT_LANGUAGE_SETTING = Setting.boolSetting("index.mapping.attachment.detect_language", false, false, Setting.Scope.INDEX);
-    public static final Setting<Integer> INDEX_ATTACHMENT_INDEXED_CHARS_SETTING = Setting.intSetting("index.mapping.attachment.indexed_chars", 100000, false, Setting.Scope.INDEX);
+    public static final Setting<Boolean> INDEX_ATTACHMENT_IGNORE_ERRORS_SETTING =
+        Setting.boolSetting("index.mapping.attachment.ignore_errors", true, Property.IndexScope);
+    public static final Setting<Boolean> INDEX_ATTACHMENT_DETECT_LANGUAGE_SETTING =
+        Setting.boolSetting("index.mapping.attachment.detect_language", false, Property.IndexScope);
+    public static final Setting<Integer> INDEX_ATTACHMENT_INDEXED_CHARS_SETTING =
+        Setting.intSetting("index.mapping.attachment.indexed_chars", 100000, Property.IndexScope);
 
     public static final String CONTENT_TYPE = "attachment";
 
diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/TikaImpl.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/TikaImpl.java
index fa9a2d06f8e..2babda8ad00 100644
--- a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/TikaImpl.java
+++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/TikaImpl.java
@@ -137,6 +137,8 @@ final class TikaImpl {
         perms.add(new SecurityPermission("putProviderProperty.BC"));
         perms.add(new SecurityPermission("insertProvider"));
         perms.add(new ReflectPermission("suppressAccessChecks"));
+        // xmlbeans, use by POI, needs to get the context classloader
+        perms.add(new RuntimePermission("getClassLoader"));
         perms.setReadOnly();
         return perms;
     }
diff --git a/plugins/mapper-attachments/src/main/plugin-metadata/plugin-security.policy b/plugins/mapper-attachments/src/main/plugin-metadata/plugin-security.policy
index e23e9f4d0cf..adf76991b59 100644
--- a/plugins/mapper-attachments/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/mapper-attachments/src/main/plugin-metadata/plugin-security.policy
@@ -27,4 +27,6 @@ grant {
   permission java.security.SecurityPermission "insertProvider";
   // TODO: fix POI XWPF to not do this: https://bz.apache.org/bugzilla/show_bug.cgi?id=58597
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+  // needed by xmlbeans, as part of POI for MS xml docs
+  permission java.lang.RuntimePermission "getClassLoader";
 };
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java
deleted file mode 100644
index 03c6e65047a..00000000000
--- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.mapper.attachments;
-
-import org.apache.commons.cli.CommandLine;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.cli.CliTool;
-import org.elasticsearch.common.cli.CliToolConfig;
-import org.elasticsearch.common.cli.Terminal;
-import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.MapperTestUtils;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.ParseContext;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Locale;
-
-import static org.elasticsearch.common.cli.CliToolConfig.Builder.cmd;
-import static org.elasticsearch.common.cli.CliToolConfig.Builder.option;
-import static org.elasticsearch.common.io.Streams.copy;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.mapper.attachments.AttachmentUnitTestCase.getIndicesModuleWithRegisteredAttachmentMapper;
-import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
-
-/**
- * This class provides a simple main class which can be used to test what is extracted from a given binary file.
- * You can run it using
- *  -u file://URL/TO/YOUR/DOC
- *  --size set extracted size (default to mapper attachment size)
- *  BASE64 encoded binary
- *
- * Example:
- *  StandaloneRunner BASE64Text
- *  StandaloneRunner -u /tmp/mydoc.pdf
- *  StandaloneRunner -u /tmp/mydoc.pdf --size 1000000
- */
-@SuppressForbidden(reason = "commandline tool")
-public class StandaloneRunner extends CliTool {
-
-    private static final CliToolConfig CONFIG = CliToolConfig.config("tika", StandaloneRunner.class)
-                        .cmds(TikaRunner.CMD)
-                .build();
-
-    static {
-        System.setProperty("es.path.home", "/tmp");
-    }
-
-    static class TikaRunner extends Command {
-        private static final String NAME = "tika";
-        private final String url;
-        private final Integer size;
-        private final String base64text;
-        private final DocumentMapper docMapper;
-
-        private static final CliToolConfig.Cmd CMD = cmd(NAME, TikaRunner.class)
-                .options(option("u", "url").required(false).hasArg(false))
-                .options(option("t", "size").required(false).hasArg(false))
-                .build();
-
-        protected TikaRunner(Terminal terminal, String url, Integer size, String base64text) throws IOException {
-            super(terminal);
-            this.size = size;
-            this.url = url;
-            this.base64text = base64text;
-            DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(PathUtils.get("."), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); // use CWD b/c it won't be used
-
-            String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/standalone/standalone-mapping.json");
-            docMapper = mapperParser.parse("person", new CompressedXContent(mapping));
-        }
-
-        @Override
-        public ExitStatus execute(Settings settings, Environment env) throws Exception {
-            XContentBuilder builder = jsonBuilder().startObject().field("file").startObject();
-
-            if (base64text != null) {
-                // If base64 is provided
-                builder.field("_content", base64text);
-            } else {
-                // A file is provided
-                byte[] bytes = copyToBytes(PathUtils.get(url));
-                builder.field("_content", bytes);
-            }
-
-            if (size >= 0) {
-                builder.field("_indexed_chars", size);
-            }
-
-            BytesReference json = builder.endObject().endObject().bytes();
-
-            ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc();
-
-            terminal.println("## Extracted text");
-            terminal.println("--------------------- BEGIN -----------------------");
-            terminal.println(doc.get("file.content"));
-            terminal.println("---------------------- END ------------------------");
-            terminal.println("## Metadata");
-            printMetadataContent(doc, AttachmentMapper.FieldNames.AUTHOR);
-            printMetadataContent(doc, AttachmentMapper.FieldNames.CONTENT_LENGTH);
-            printMetadataContent(doc, AttachmentMapper.FieldNames.CONTENT_TYPE);
-            printMetadataContent(doc, AttachmentMapper.FieldNames.DATE);
-            printMetadataContent(doc, AttachmentMapper.FieldNames.KEYWORDS);
-            printMetadataContent(doc, AttachmentMapper.FieldNames.LANGUAGE);
-            printMetadataContent(doc, AttachmentMapper.FieldNames.NAME);
-            printMetadataContent(doc, AttachmentMapper.FieldNames.TITLE);
-
-            return ExitStatus.OK;
-        }
-
-        private void printMetadataContent(ParseContext.Document doc, String field) {
-            terminal.println("- " + field + ":" + doc.get(docMapper.mappers().getMapper("file." + field).fieldType().name()));
-        }
-
-        public static byte[] copyToBytes(Path path) throws IOException {
-            try (InputStream is = Files.newInputStream(path);
-                 BytesStreamOutput out = new BytesStreamOutput()) {
-                copy(is, out);
-                return out.bytes().toBytes();
-            }
-        }
-
-        public static Command parse(Terminal terminal, CommandLine cli) throws IOException {
-            String url = cli.getOptionValue("u");
-            String base64text = null;
-            String sSize = cli.getOptionValue("size");
-            Integer size = sSize != null ? Integer.parseInt(sSize) : -1;
-            if (url == null && cli.getArgs().length == 0) {
-                    return exitCmd(ExitStatus.USAGE, terminal, "url or BASE64 content should be provided (type -h for help)");
-            }
-            if (url == null) {
-                if (cli.getArgs().length == 0) {
-                    return exitCmd(ExitStatus.USAGE, terminal, "url or BASE64 content should be provided (type -h for help)");
-                }
-                base64text = cli.getArgs()[0];
-            } else {
-                if (cli.getArgs().length == 1) {
-                    return exitCmd(ExitStatus.USAGE, terminal, "url or BASE64 content should be provided. Not both. (type -h for help)");
-                }
-            }
-            return new TikaRunner(terminal, url, size, base64text);
-        }
-    }
-
-    public StandaloneRunner() {
-        super(CONFIG);
-    }
-
-
-    public static void main(String[] args) throws Exception {
-        StandaloneRunner pluginManager = new StandaloneRunner();
-        pluginManager.execute(args);
-    }
-
-    @Override
-    protected Command parse(String cmdName, CommandLine cli) throws Exception {
-        switch (cmdName.toLowerCase(Locale.ROOT)) {
-            case TikaRunner.NAME: return TikaRunner.parse(terminal, cli);
-            default:
-                    assert false : "can't get here as cmd name is validated before this method is called";
-                    return exitCmd(ExitStatus.CODE_ERROR);
-        }
-    }
-}
diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml
new file mode 100644
index 00000000000..69991b9d0c0
--- /dev/null
+++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml
@@ -0,0 +1,78 @@
+setup:
+    - do:
+        indices.create:
+            index: test
+            body:
+              mappings:
+                test:
+                  properties:
+                    file:
+                      type: attachment
+                      fields:
+                        content:
+                          store: true
+                        author:
+                          store: true
+                        date:
+                          store: true
+                        content_length:
+                          store: true
+                        content_type:
+                          store: true
+                        
+    - do:
+        cluster.health:
+          wait_for_status: yellow
+---
+"Test mapper attachment processor with .doc file":
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        refresh: true
+        body: { file: "0M8R4KGxGuEAAAAAAAAAAAAAAAAAAAAAPgADAP7/CQAGAAAAAAAAAAAAAAAEAAAAjAEAAAAAAAAAEAAAjgEAAAEAAAD+////AAAAAIgBAACJAQAAigEAAIsBAAD////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////spcEAg+kMBAAA8BK/AAAAAAABEQABAAEACAAAEwgAAA4AYmpiaoI4gjgAAAAAAAAAAAAAAAAAAAAAAAAMBBYANA4AAOBSAADgUgAAEwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD//w8AAAAAAAAAAAD//w8AAAAAAAAAAAD//w8AAAAAAAAAAAAAAAAAAAAAALcAAAAAAFAHAAAAAAAAUAcAAMcUAAAAAAAAxxQAAAAAAADHFAAAAAAAAMcUAAAAAAAAxxQAABQAAAAAAAAAAAAAAP////8AAAAA2xQAAAAAAADbFAAAAAAAANsUAAAAAAAA2xQAAAwAAADnFAAADAAAANsUAAAAAAAA3hUAADABAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAAVRUAAAIAAABXFQAAAAAAAFcVAAAAAAAAVxUAAAAAAABXFQAAAAAAAFcVAAAAAAAAVxUAACwAAAAOFwAAtgIAAMQZAABaAAAAgxUAABUAAAAAAAAAAAAAAAAAAAAAAAAAxxQAAAAAAADzFAAAAAAAAAAAAAAAAAAAAAAAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAIMVAAAAAAAAGRUAAAAAAADHFAAAAAAAAMcUAAAAAAAA8xQAAAAAAAAAAAAAAAAAAPMUAAAAAAAAmBUAABYAAAAZFQAAAAAAABkVAAAAAAAAGRUAAAAAAADzFAAAFgAAAMcUAAAAAAAA8xQAAAAAAADHFAAAAAAAAPMUAAAAAAAAVRUAAAAAAAAAAAAAAAAAABkVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA8xQAAAAAAABVFQAAAAAAAAAAAAAAAAAAGRUAAAAAAAAAAAAAAAAAABkVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRUAAAAAAAAAAAAAAAAAAP////8AAAAAgI6XYKZ60QEAAAAAAAAAAP////8AAAAACRUAABAAAAAZFQAAAAAAAAAAAAAAAAAAQRUAABQAAACuFQAAMAAAAN4VAAAAAAAAGRUAAAAAAAAeGgAAAAAAABkVAAAAAAAAHhoAAAAAAAAZFQAAAAAAABkVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADHFAAAAAAAABkVAAAoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAAgxUAAAAAAACDFQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAN4VAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAAAAAAAAAAAAAP////8AAAAA/////wAAAAD/////AAAAAAAAAAAAAAAA/////wAAAAD/////AAAAAP////8AAAAA/////wAAAAD/////AAAAAP////8AAAAA/////wAAAAD/////AAAAAP////8AAAAA/////wAAAAD/////AAAAAP////8AAAAA/////wAAAAD/////AAAAAB4aAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAADzFAAAAAAAAPMUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADzFAAAAAAAAPMUAAAAAAAA8xQAAAAAAABQBwAAPQwAAI0TAAA6AQAABwAMAQ8ADQEAAAwEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFRlc3QgZWxhc3RpY3NlYXJjaA0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAABIIAAATCAAA/PgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYWaJVGuQAABhZo3wiGAAIACAAAEwgAAP0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAATIAMZBoATpwpBeqAB+wfC4gsMhBIbCJBSKwiQUjkIkFJJCJBSWwAAAXsMQCGLDEAgyQxAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALgYPABIAAQB8AQ8ACAADAAMAAwAAAAQACAAAAJgAAACeAAAAngAAAJ4AAACeAAAAngAAAJ4AAACeAAAAngAAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAAHYCAAB2AgAAdgIAAHYCAAB2AgAAdgIAAHYCAAB2AgAAdgIAADYGAAA2BgAANgYAADYGAAA2BgAANgYAAD4CAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAACoAAAANgYAADYGAAAWAAAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAC4AAAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAAaAEAAEgBAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAAHACAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAANgYAADYGAAA2BgAAMgYAABgAAADGAwAA1gMAAOYDAAD2AwAABgQAABYEAAAmBAAANgQAAEYEAABWBAAAZgQAAHYEAACGBAAAlgQAAMYDAADWAwAA5gMAAPYDAAAGBAAAFgQAADIGAAAoAgAA2AEAAOgBAAAmBAAANgQAAEYEAABWBAAAZgQAAHYEAACGBAAAlgQAAMYDAADWAwAA5gMAAPYDAAAGBAAAFgQAACYEAAA2BAAARgQAAFYEAABmBAAAdgQAAIYEAACWBAAAxgMAANYDAADmAwAA9gMAAAYEAAAWBAAAJgQAADYEAABGBAAAVgQAAGYEAAB2BAAAhgQAAJYEAADGAwAA1gMAAOYDAAD2AwAABgQAABYEAAAmBAAANgQAAEYEAABWBAAAZgQAAHYEAACGBAAAlgQAAMYDAADWAwAA5gMAAPYDAAAGBAAAFgQAACYEAAA2BAAARgQAAFYEAABmBAAAdgQAAIYEAACWBAAAxgMAANYDAADmAwAA9gMAAAYEAAAWBAAAJgQAADYEAABGBAAAVgQAAGYEAAB2BAAAhgQAAJYEAAA4AQAAWAEAAPgBAAAIAgAAGAIAAFYCAAB+AgAAkAIAAKACAACwAgAAwAIAANACAACAAgAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAADgAgAA8AIAAAADAAAQAwAAIAMAADADAABAAwAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAADgAgAA8AIAAAADAAAQAwAAIAMAADADAABAAwAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAADgAgAA8AIAAAADAAAQAwAAIAMAADADAABAAwAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAADgAgAA8AIAAAADAAAQAwAAIAMAADADAABAAwAA4AIAAPACAAAAAwAAEAMAACADAAAwAwAAQAMAAOACAADwAgAAAAMAABADAAAgAwAAMAMAAEADAAAgAAAAT0oDAFBKAwBRSgMAX0gBBG1IDARuSAwEc0gMBHRIDAQAAAAAQAAAYPH/AgBAAAwQAAAAAAAAAAAGAE4AbwByAG0AYQBsAAAAAgAAABgAQ0oYAF9IAQRhShgAbUgMBHNIDAR0SAkEAAAAAAAAAAAAAAAAAAAAAAAAOgBBIPL/oQA6AAwNAAAAAAAAEAARAFAAbwBsAGkAYwBlACAAcABhAHIAIABkAOkAZgBhAHUAdAAAAAAAVgBpAPP/swBWAAwNAAAAAAAAMAYOAFQAYQBiAGwAZQBhAHUAIABOAG8AcgBtAGEAbAAAABwAF/YDAAA01gYAAQoDbAA01gYAAQUDAABh9gMAAAIACwAAADIAayD0/8EAMgAADQAAAAAAADAGDABBAHUAYwB1AG4AZQAgAGwAaQBzAHQAZQAAAAIADAAAAAAAUEsDBBQABgAIAAAAIQCb6HBP/AAAABwCAAATAAAAW0NvbnRlbnRfVHlwZXNdLnhtbKyRy2rDMBBF94X+g9C22HK6KKXYzqKPXR+L9AMGeWyL2CMhTULy9x07LpQSAoVuBNLMvffMqFwfxkHtMSbnqdKrvNAKyfrGUVfpz81Ldq9VYqAGBk9Y6SMmva6vr8rNMWBSoqZU6Z45PBiTbI8jpNwHJKm0Po7Aco2dCWC30KG5LYo7Yz0xEmc8eei6fMIWdgOr54M8n0hErtXjqW+KqjSEMDgLLKBmqpqzuohDuiDcU/OLLlvIclHO5ql3Id0sCe+ymugaVB8Q+Q1G4TAsQ+LP8xVIRov5ZeYz0b5tncXG290o68hn48XsTwCr/4n+zjTz39ZfAAAA//8DAFBLAwQUAAYACAAAACEApdan58AAAAA2AQAACwAAAF9yZWxzLy5yZWxzhI/PasMwDIfvhb2D0X1R0sMYJXYvpZBDL6N9AOEof2giG9sb69tPxwYKuwiEpO/3qT3+rov54ZTnIBaaqgbD4kM/y2jhdj2/f4LJhaSnJQhbeHCGo3vbtV+8UNGjPM0xG6VItjCVEg+I2U+8Uq5CZNHJENJKRds0YiR/p5FxX9cfmJ4Z4DZM0/UWUtc3YK6PqMn/s8MwzJ5PwX+vLOVFBG43lExp5GKhqC/jU72QqGWq1B7Qtbj51v0BAAD//wMAUEsDBBQABgAIAAAAIQBreZYWgwAAAIoAAAAcAAAAdGhlbWUvdGhlbWUvdGhlbWVNYW5hZ2VyLnhtbAzMTQrDIBBA4X2hd5DZN2O7KEVissuuu/YAQ5waQceg0p/b1+XjgzfO3xTVm0sNWSycBw2KZc0uiLfwfCynG6jaSBzFLGzhxxXm6XgYybSNE99JyHNRfSPVkIWttd0g1rUr1SHvLN1euSRqPYtHV+jT9yniResrJgoCOP0BAAD//wMAUEsDBBQABgAIAAAAIQBtTVmryAYAAI4aAAAWAAAAdGhlbWUvdGhlbWUvdGhlbWUxLnhtbOxZ3YrbRhS+L/QdhO4d/0n+WeINtmxv2uwmIXbS5nJWHkuTHWmMZrwbEwJ9gkIhLb0p9K6F3gTaN+i7pLTpQ/TMSJZn7HH2hy2E0jUs8vg7Z7455+g7I83dey8T6pzjjBOW9tz6nZrr4DRkM5JGPffpdFzpuA4XKJ0hylLcc1eYu/cOP/3kLjoQMU6wA/YpP0A9NxZicVCt8hCGEb/DFjiF3+YsS5CAr1lUnWXoAvwmtNqo1VrVBJHUdVKUgNtp/PvP4OzRfE5C7B6uvY8oTJEKLgdCmk2kb1yYDJYZRkuFnZ3VJYKveEAz5xzRngsTzdjFFL8UrkMRF/BDz62pP7d6eLeKDgojKvbYanZj9VfYFQazs4aaM4tOy0k9z/da/dK/AlCxixu1R61Rq/SnACgMYaU5F92nP+gOhn6B1UD5pcX3sD1s1g285r+5w7nvy4+BV6Dcv7eDH48DiKKBV6Ac7+/gPa/dCDwDr0A5vrWDb9f6Q69t4BUopiQ920HX/FYzWK+2hMwZvW+Fd31v3G4UzjcoqIayuuQUc5aKfbWWoBcsGwNAAikSJHXEaoHnKIQyDhAlpxlxjkkUQ+EtUMo4DNcatXGtCf/lx1NXKiLoACPNWvICJnxnSPJxeJiRhei5n4NXV4M8XzpHTMQkLGZVTgyL+yiNdIv3P33z9w9fOX/9+uP7N9/mk27juY4f4jT6kqD0QxPAajdhePfd2z9+e/vu+6///OWNxX8/Q6c6fEoSzJ2H+MJ5whJYnGUF+DS7nsU0RkS36KcRRymSs1j8jyB+OvrhClFkwQ0gEjruWQYyYwMeLV8YhCdxthTE4vFBnBjAE8bogGXWKDyQc2lhni7TyD55ttRxTxA6t80doNTI82i5AH0lNpdBjA2ajylKBYpwioUjf2NnGFtW95wQI64nJMwYZ3PhPCfOABFrSKbk1KimjdF9kkBeVjaCkG8jNifPnAGjtlUP8bmJhLsDUQv5KaZGGI/QUqDE5nKKEqoH/BiJ2EZysspCHTfiAjIdYcqc0QxzbrN5lMF6taQ/AImxp/2ErhITmQlyZvN5jBjTkUN2FsQoWdiwE5LGOvYzfgYlipzHTNjgJ8y8Q+R3yAOIx750PyPYSPflavAU1FWntCkQ+csys+TyCDOjficrOkdYSQ2Iv6HpCUkvFfgtaff/PWk/IWkYM8uKbkvU7a6NjFxTzvsZsd5P97dEfB9uW7oDls3Ix6/cQ7RMH2O4WXbb1//C/b9wu/954d53P9++XG8UGsRbbl3zzbrauid7d+5zQulErCg+5mrzzqEvzcYwKO3UYysun+QWMVzKOxkmMHBRhpSNkzHxBRHxJEYL2OHXXekk4oXriDsLxmHjr4atviWeLpMTNssfWOt1+XCaiwdHYjNe88txeNgQObrV3jyEle4V20g9LK8JSNvrkNAmM0k0LSTa60EZJPVoDkGzkFAruxUWXQuLjnS/TtUOC6BWZgU2Tg5st3qu74EJGMEzFaJ4JvOUp3qdXZXM28z0vmAaFQC7iHUFbDLdlVz3Lk+uLi+1K2TaIKGVm0lCRUb1MB6jGS6qU45ehcZ1c93dpNSgJ0Oh5oPS2tBodz7E4qa5BrttbaCprhQ0dS56bqvpQ8mEaNFz5/DgD5fJAmqHyw0vohG8PgtFlt/wN1GWRcbFEPE4D7gSnVwNEiJw5lCS9Fy5/DINNFUaorjVGyAIHy25LsjKx0YOkm4mGc/nOBR62rURGen8Kyh8rhXWX5X5zcHSki0h3ZN4duGc0mX2BEGJ+e26DOCMcHj/U8+jOSPwQrMUsk39bTWmQnb1N4qqhvJxRBcxKjqKLuY5XEl5SUd9K2OgfSvWDAHVQlI0wtNINlg9qEY3LbtGzmFv173cSEZOE81NzzRURXZNu4oZM6zbwFYsb9bkNVbrEIOm6R0+l+5tye2utW5rn1B2CQh4GT9L171CQ9CobSYzqEnGuzIsNbsYNXvHeoGXULtKk9BUv7V2uxW3skdYp4PBG3V+sNuuWhiar/eVKtLq6EM/nGCnL0A8hvAaeEkFV6mEo4cMwYZoovYkuWzALfJSFLcGXDnLjPTcVzW/7wUNP6jUOv6o4jW9WqXj95uVvu836yO/XhsOGq+hsYg4qfv5scsYXkTRVXH4osZ3DmCS9bu2OyFLqkydrFQVcXUAU28YBzD5yYszlQcsrkNAdF61GuNusztoVbrN/rjiDQedSjdoDSrDVtAejoeB3+mOX7vOuQJ7/WbgtUadSqseBBWvVZP0O91K22s0+l673xl5/dfFNgZWnstHEQsIr+J1+A8AAAD//wMAUEsDBBQABgAIAAAAIQAN0ZCftgAAABsBAAAnAAAAdGhlbWUvdGhlbWUvX3JlbHMvdGhlbWVNYW5hZ2VyLnhtbC5yZWxzhI9NCsIwFIT3gncIb2/TuhCRJt2I0K3UA4TkNQ02PyRR7O0NriwILodhvplpu5edyRNjMt4xaKoaCDrplXGawW247I5AUhZOidk7ZLBggo5vN+0VZ5FLKE0mJFIoLjGYcg4nSpOc0IpU+YCuOKOPVuQio6ZByLvQSPd1faDxmwF8xSS9YhB71QAZllCa/7P9OBqJZy8fFl3+UUFz2YUFKKLGzOAjm6pMBMpburrE3wAAAP//AwBQSwECLQAUAAYACAAAACEAm+hwT/wAAAAcAgAAEwAAAAAAAAAAAAAAAAAAAAAAW0NvbnRlbnRfVHlwZXNdLnhtbFBLAQItABQABgAIAAAAIQCl1qfnwAAAADYBAAALAAAAAAAAAAAAAAAAAC0BAABfcmVscy8ucmVsc1BLAQItABQABgAIAAAAIQBreZYWgwAAAIoAAAAcAAAAAAAAAAAAAAAAABYCAAB0aGVtZS90aGVtZS90aGVtZU1hbmFnZXIueG1sUEsBAi0AFAAGAAgAAAAhAG1NWavIBgAAjhoAABYAAAAAAAAAAAAAAAAA0wIAAHRoZW1lL3RoZW1lL3RoZW1lMS54bWxQSwECLQAUAAYACAAAACEADdGQn7YAAAAbAQAAJwAAAAAAAAAAAAAAAADPCQAAdGhlbWUvdGhlbWUvX3JlbHMvdGhlbWVNYW5hZ2VyLnhtbC5yZWxzUEsFBgAAAAAFAAUAXQEAAMoKAAAAADw/eG1sIHZlcnNpb249IjEuMCIgZW5jb2Rpbmc9IlVURi04IiBzdGFuZGFsb25lPSJ5ZXMiPz4NCjxhOmNsck1hcCB4bWxuczphPSJodHRwOi8vc2NoZW1hcy5vcGVueG1sZm9ybWF0cy5vcmcvZHJhd2luZ21sLzIwMDYvbWFpbiIgYmcxPSJsdDEiIHR4MT0iZGsxIiBiZzI9Imx0MiIgdHgyPSJkazIiIGFjY2VudDE9ImFjY2VudDEiIGFjY2VudDI9ImFjY2VudDIiIGFjY2VudDM9ImFjY2VudDMiIGFjY2VudDQ9ImFjY2VudDQiIGFjY2VudDU9ImFjY2VudDUiIGFjY2VudDY9ImFjY2VudDYiIGhsaW5rPSJobGluayIgZm9sSGxpbms9ImZvbEhsaW5rIi8+AAAAABMAAAAUAAAOAAAIAP////8ACAAAEwgAAAUAAAAACAAAEwgAAAYAAAAAAAAABQAAABIAAAAVAAAABwAEAAcAAAAAABIAAAAVAAAABAAHAAQAAAAEAAAACAAAAOUAAAAAAAAAAwAAAN8IhgCkF6oAlUa5AH419AAAAAAAEwAAABUAAAAAAAAAAQAAAP9AAIABABIAAAASAAAAAEBDewEAAQASAAAAAAAAABIAAAAAAAAAAAAAAAAAAAACEAAAAAAAAAATAAAAoAAAEABAAAD//wEAAAAHAFUAbgBrAG4AbwB3AG4A//8BAAgAAAAAAAAAAAAAAP//AQAAAAAA//8AAAIA//8AAAAA//8AAAIA//8AAAAABQAAAEcOkAEAAAICBgMFBAUCAwTvKgDgQXgAwAkAAAAAAAAA/wEAAAAAAABUAGkAbQBlAHMAIABOAGUAdwAgAFIAbwBtAGEAbgAAADUOkAECAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAgAAAAABTAHkAbQBiAG8AbAAAADMOkAEAAAILBgQCAgICAgT/KgDgQ3gAwAkAAAAAAAAA/wEAAAAAAABBAHIAaQBhAGwAAAA3DpABAAACDwUCAgIEAwIE/wIA4P+sAEABAAAAAAAAAJ8BAAAAAAAAQwBhAGwAaQBiAHIAaQAAAEESkAEBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABDAGEAbQBiAHIAaQBhACAATQBhAHQAaAAAACAABADxCIgIAPDEAgAAqQEAAAAAWVJDh1lSQ4cAAAAAAgABAAAAAgAAABEAAAABAAEAAAAEAAOQAQAAAAIAAAARAAAAAQABAAAAAQAAAAAAAAAhAwDwEAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAClBsAHtAC0AIGBcjAAAAAAAAAAAAAAAAAAABIAAAASAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAABAAAAA8BAACAD8/QEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACSFAAAAAACfH/DwAAJFAAABAnAAD///9/////f////3////9/////f////3////9/3wiGAAAEAAAyAAAAAAAAAAAAAAAAAAAAAAAAAAAAIQQAAAAAAAAAAAAAAAAAAAAAAAAQHAAABAAAAAAAAAAAAHgAAAB4AAAAAAAAAAAAAACgBQAAGkjOCAsAAAAAAAAA3AAAAAEAAAD//xIAAAAAAAAAAAAAAAAAAAAMAEQAYQB2AGkAZAAgAFAAaQBsAGEAdABvAAwARABhAHYAaQBkACAAUABpAGwAYQB0AG8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP7/AAADCgEAAAAAAAAAAAAAAAAAAAAAAAEAAADghZ/y+U9oEKuRCAArJ7PZMAAAANzSAgASAAAAAQAAAJgAAAACAAAAoAAAAAMAAACsAAAABAAAALgAAAAFAAAA0AAAAAYAAADcAAAABwAAAOgAAAAIAAAA/AAAAAkAAAAUAQAAEgAAACABAAAKAAAARAEAAAwAAABQAQAADQAAAFwBAAAOAAAAaAEAAA8AAABwAQAAEAAAAHgBAAATAAAAgAEAABEAAACIAQAAAgAAABAnAAAeAAAABAAAAAAAAAAeAAAABAAAAAAAAAAeAAAAEAAAAERhdmlkIFBpbGF0bwAAAAAeAAAABAAAAAAAAAAeAAAABAAAAAAAAAAeAAAADAAAAE5vcm1hbC5kb3RtAB4AAAAQAAAARGF2aWQgUGlsYXRvAAAAAB4AAAAEAAAAMgAAAB4AAAAcAAAATWljcm9zb2Z0IE1hY2ludG9zaCBXb3JkAAAAAEAAAAAARsMjAAAAAEAAAAAAFjZWpnrRAUAAAAAAFjZWpnrRAQMAAAABAAAAAwAAAAIAAAADAAAAEQAAAAMAAAAAAAAARwAAAEzRAgD/////DgAAAAEAAABsAAAAAAAAAAAAAAD/AAAAswAAAAAAAAAAAAAAZhkAANsRAAAgRU1GAAABAETRAgAIAAAAAQAAAAAAAAAAAAAAAAAAAOwEAACxAwAAQAEAAPAAAAAAAAAAAAAAAAAAAAAA4gQAgKkDABEAAAAMAAAACAAAAAoAAAAQAAAAAAAAAAAAAAAJAAAAEAAAAAABAAC0AAAADAAAABAAAAAAAAAAAAAAAAsAAAAQAAAAAAEAALQAAABRAAAAeNACAAAAAAAAAAAA/wAAALMAAAAAAAAAAAAAAAAAAAAAAAAAAAEAALQAAABQAAAAKAAAAHgAAAAA0AIAAAAAACAAzAAAAQAAtAAAACgAAAAAAQAAtAAAAAEAIAAAAAAAANACAAAAAAAAAAAAAAAAAAAAAAD/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////vr6+/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/76+vv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////7vf//+rz7v/Yzc3/0NLY/+DX2f/N4PL/3tXI/8jV4v/Q0cX/1tDI/9ve2f/U0tX/0NLQ/83I0P/I2N7/4tnI/9LZ4v/v6tz/5eXl////9////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////83g9//e3M3/vrG3/8TCxv/Xwrz/vdfu/8W/rv/K1tX/x8bB/8LJxv/Oxb7/yMTE/8vCwv+3scH/zd7Z/9DNyP/BwcT/z97X/82xq/////v////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////u9/v/+/Lu////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////zs7O/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////87Ozv/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////Ozs7///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////++vr7/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/5OTk/+Tk5P/k5OT/vr6+//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8OAAAAFAAAAAAAAAAQAAAAFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD+/wAAAwoBAAAAAAAAAAAAAAAAAAAAAAABAAAAAtXN1ZwuGxCTlwgAKyz5rjAAAADUAAAACwAAAAEAAABgAAAABQAAAGgAAAAGAAAAcAAAABEAAAB4AAAAFwAAAIAAAAALAAAAiAAAABAAAACQAAAAEwAAAJgAAAAWAAAAoAAAAA0AAACoAAAADAAAALUAAAACAAAAECcAAAMAAAABAAAAAwAAAAEAAAADAAAAEgAAAAMAAAAAAA8ACwAAAAAAAAALAAAAAAAAAAsAAAAAAAAACwAAAAAAAAAeEAAAAQAAAAEAAAAADBAAAAIAAAAeAAAABgAAAFRpdHJlAAMAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAIAAAADAAAABAAAAAUAAAAGAAAABwAAAP7///8JAAAACgAAAAsAAAAMAAAADQAAAA4AAAAPAAAAEAAAABEAAAASAAAAEwAAABQAAAAVAAAA/v///xcAAAAYAAAAGQAAABoAAAAbAAAAHAAAAB0AAAAeAAAAHwAAACAAAAAhAAAAIgAAACMAAAAkAAAAJQAAACYAAAAnAAAAKAAAACkAAAAqAAAAKwAAACwAAAAtAAAALgAAAC8AAAAwAAAAMQAAADIAAAAzAAAANAAAADUAAAA2AAAANwAAADgAAAA5AAAAOgAAADsAAAA8AAAAPQAAAD4AAAA/AAAAQAAAAEEAAABCAAAAQwAAAEQAAABFAAAARgAAAEcAAABIAAAASQAAAEoAAABLAAAATAAAAE0AAABOAAAATwAAAFAAAABRAAAAUgAAAFMAAABUAAAAVQAAAFYAAABXAAAAWAAAAFkAAABaAAAAWwAAAFwAAABdAAAAXgAAAF8AAABgAAAAYQAAAGIAAABjAAAAZAAAAGUAAABmAAAAZwAAAGgAAABpAAAAagAAAGsAAABsAAAAbQAAAG4AAABvAAAAcAAAAHEAAAByAAAAcwAAAHQAAAB1AAAAdgAAAHcAAAB4AAAAeQAAAHoAAAB7AAAAfAAAAH0AAAB+AAAAfwAAAIAAAACBAAAAggAAAIMAAACEAAAAhQAAAIYAAACHAAAAiAAAAIkAAACKAAAAiwAAAIwAAACNAAAAjgAAAI8AAACQAAAAkQAAAJIAAACTAAAAlAAAAJUAAACWAAAAlwAAAJgAAACZAAAAmgAAAJsAAACcAAAAnQAAAJ4AAACfAAAAoAAAAKEAAACiAAAAowAAAKQAAAClAAAApgAAAKcAAACoAAAAqQAAAKoAAACrAAAArAAAAK0AAACuAAAArwAAALAAAACxAAAAsgAAALMAAAC0AAAAtQAAALYAAAC3AAAAuAAAALkAAAC6AAAAuwAAALwAAAC9AAAAvgAAAL8AAADAAAAAwQAAAMIAAADDAAAAxAAAAMUAAADGAAAAxwAAAMgAAADJAAAAygAAAMsAAADMAAAAzQAAAM4AAADPAAAA0AAAANEAAADSAAAA0wAAANQAAADVAAAA1gAAANcAAADYAAAA2QAAANoAAADbAAAA3AAAAN0AAADeAAAA3wAAAOAAAADhAAAA4gAAAOMAAADkAAAA5QAAAOYAAADnAAAA6AAAAOkAAADqAAAA6wAAAOwAAADtAAAA7gAAAO8AAADwAAAA8QAAAPIAAADzAAAA9AAAAPUAAAD2AAAA9wAAAPgAAAD5AAAA+gAAAPsAAAD8AAAA/QAAAP4AAAD/AAAAAAEAAAEBAAACAQAAAwEAAAQBAAAFAQAABgEAAAcBAAAIAQAACQEAAAoBAAALAQAADAEAAA0BAAAOAQAADwEAABABAAARAQAAEgEAABMBAAAUAQAAFQEAABYBAAAXAQAAGAEAABkBAAAaAQAAGwEAABwBAAAdAQAAHgEAAB8BAAAgAQAAIQEAACIBAAAjAQAAJAEAACUBAAAmAQAAJwEAACgBAAApAQAAKgEAACsBAAAsAQAALQEAAC4BAAAvAQAAMAEAADEBAAAyAQAAMwEAADQBAAA1AQAANgEAADcBAAA4AQAAOQEAADoBAAA7AQAAPAEAAD0BAAA+AQAAPwEAAEABAABBAQAAQgEAAEMBAABEAQAARQEAAEYBAABHAQAASAEAAEkBAABKAQAASwEAAEwBAABNAQAATgEAAE8BAABQAQAAUQEAAFIBAABTAQAAVAEAAFUBAABWAQAAVwEAAFgBAABZAQAAWgEAAFsBAABcAQAAXQEAAF4BAABfAQAAYAEAAGEBAABiAQAAYwEAAGQBAABlAQAAZgEAAGcBAABoAQAAaQEAAGoBAABrAQAAbAEAAG0BAABuAQAAbwEAAHABAABxAQAAcgEAAHMBAAB0AQAAdQEAAHYBAAB3AQAAeAEAAHkBAAB6AQAAewEAAHwBAAB9AQAAfgEAAH8BAAD+////gQEAAIIBAACDAQAAhAEAAIUBAACGAQAAhwEAAP7////9/////f////3////9////jQEAAP7////+/////v////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////9SAG8AbwB0ACAARQBuAHQAcgB5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFgAFAf//////////AwAAAAYJAgAAAAAAwAAAAAAAAEYAAAAAAAAAAAAAAAAgFZlgpnrRAY8BAACAAAAAAAAAADEAVABhAGIAbABlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOAAIB/////wUAAAD/////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAB4aAAAAAAAAVwBvAHIAZABEAG8AYwB1AG0AZQBuAHQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABoAAgEBAAAA//////////8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAFAFMAdQBtAG0AYQByAHkASQBuAGYAbwByAG0AYQB0AGkAbwBuAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAACAQIAAAAEAAAA/////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABYAAAAM0wIAAAAAAAUARABvAGMAdQBtAGUAbgB0AFMAdQBtAG0AYQByAHkASQBuAGYAbwByAG0AYQB0AGkAbwBuAAAAAAAAAAAAAAA4AAIB////////////////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAEAAAAQAAAAAAAAAQBDAG8AbQBwAE8AYgBqAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABIAAgD///////////////8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAcgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP///////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA////////////////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAP7///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8BAP7/AwoAAP////8GCQIAAAAAAMAAAAAAAABGIAAAAERvY3VtZW50IE1pY3Jvc29mdCBXb3JkIDk3LTIwMDQACgAAAE1TV29yZERvYwAQAAAAV29yZC5Eb2N1bWVudC44APQ5snEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" }
+
+  - do:
+      search:
+          index: test
+          body:
+              fields: [file.content, file.author, file.date, file.content_length, file.content_type]
+  - match: { hits.total: 1 }
+  - match: { hits.hits.0.fields: {
+      file.content: ["Test elasticsearch\n"],
+      file.author: ["David Pilato"],
+      file.date: ["2016-03-10T08:25:00Z"],
+      file.content_length: ["205312"],
+      file.content_type: ["application/msword"]
+    }
+  }
+
+
+---
+"Test mapper attachment processor with .docx file":
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        refresh: true
+        body: { file: "UEsDBBQABgAIAAAAIQBtiidLZgEAAFQFAAATAAgCW0NvbnRlbnRfVHlwZXNdLnhtbCCiBAIooAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC0lMtugzAQRfeV+g/I2wqcdFFVVUgWfSzbSE0/wLEH4tYv2c7r7ztAgqooAalJNkgwc+89A3hGk41WyQp8kNbkZJgNSAKGWyFNmZOv2Vv6SJIQmRFMWQM52UIgk/HtzWi2dRASVJuQk0WM7onSwBegWcisA4OVwnrNIt76kjrGf1gJ9H4weKDcmggmprHyIOPRCxRsqWLyusHHDQnKSfLc9FVROWHOKclZxDKtqvSozoMKHcKVEQd06Y4sQ2XdExbShbvTCd8OyoMEqavR6gJqPvB1eikgmTIf35nGBrq2XlBh+VKjKOse7gijLQrJodVXbs5bDiHgd9IqayuaSbNnP8kR4lZBuDxF49sfDzGi4BoAO+dehDXMP69G8ce8F6TA3BmbK7g8RmvdCxHx1EJzHZ7NUdt0RWLn1FsXcAv4f4y9P66VOsWBHfgou/+6NhGtz54Pqk0gQBzJpvVOHP8CAAD//wMAUEsDBBQABgAIAAAAIQDHwie8/wAAAN8CAAALAAgCX3JlbHMvLnJlbHMgogQCKKAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAArJLNSgMxEIDvgu8Q5t7NtoqINNuLCL2JrA8wJtPd6OaHZKrt2xtF1IVlEexx/j6+SWa9ObhBvFLKNngFy6oGQV4HY32n4LG9W1yDyIze4BA8KThShk1zfrZ+oAG5DOXexiwKxWcFPXO8kTLrnhzmKkTypbILySGXMHUyon7BjuSqrq9k+s2AZsQUW6Mgbc0FiPYY6X9s6YjRIKPUIdEipjKd2JZdRIupI1Zggr4v6fzZURUyyGmhy78Lhd3OaroNeu/I85QXHZi8ITOvhDHOGS1PaTTu+JF5C8lI85Wes1md9sO437snj3aYeJfvWvUcqfsQkqOzbN4BAAD//wMAUEsDBBQABgAIAAAAIQATqj6H9gAAADEDAAAcAAgBd29yZC9fcmVscy9kb2N1bWVudC54bWwucmVscyCiBAEooAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKySy2rDMBBF94X+g5h9LTt9UELkbEoh29b9AEUeP6gsCc304b+vaEjr0GC68PJeMfeeQbPZfg5WvGOk3jsFRZaDQGd83btWwUv1eHUPgli7WlvvUMGIBNvy8mLzhFZzGqKuDyRSiiMFHXNYS0mmw0FT5gO69NL4OGhOMrYyaPOqW5SrPL+TcZoB5Umm2NUK4q6+BlGNAf+T7ZumN/jgzduAjs9UyA/cPyNzWo5SrI4tsoKJmaVEkOdBbpYEabzjSu8t/mL8WHMQt0tCcJqdAHzLg1nMMRRLMhCPFiefcdBz9atF6/9cw9E5IsiTQy+/AAAA//8DAFBLAwQUAAYACAAAACEA9WKOYGUCAAAOBwAAEQAAAHdvcmQvZG9jdW1lbnQueG1spFXfb9owEH6ftP8h8jtNwijQiFDR0qI+TKpK9zwZx0ksYp9lGyj763dOIGSbVtGSh9j367vv7mJncvsmq2DLjRWgUhJfRSTgikEmVJGSH6+PvTEJrKMqoxUonpI9t+R2+vXLZJdkwDaSKxcghLLJTrOUlM7pJAwtK7mk9koKZsBC7q4YyBDyXDAe7sBkYT+Ko3qnDTBuLea7p2pLLTnASTgPTVJ23PajaIyyUC3Gv4xAc4XGHIykDkVTYIRZb3QPMTV1YiUq4fYea9jCbFOyMSo5YPRaHj4mQQLJVlZHZ3jPtyF6WI4R5hySTcj80PKaXmh4hYRB2VLoU98+i4bG8gjybsGdYnc6Hlw29LmhO1xOgOfQz5ogWTXM30eMozMm4iHaiHMo/JnzyKT78e0+15pOc+PrjwH0/wbQxWXDWRjY6BOauAztSa1bLH+VfADrMORuafYyMsuSajyBkiVPhQJDVxUywpEF2PXAf9ZkilfcCrK9XzWqB4mmhj5lKRmNhg/X9/GI1FrH31yjbR7UJnidZi8piaK7m8Hw5rpVzXlON5XzlvEwGs8f6yzGv9z0lVsX4JG2TjDLqWHlJPR6/65dVgBrf1ktHTUOIQVmjTy2ohLZ/1zAHWVrEnZ9H1TWeoY1lPZmy5l7Nv9nukS7185m8WjW9EIXy19oxdMRxzdRnbfE/XA8qJG9w3fqIR3gIY4HdX8SI4rSncQVOAfyJFc871hLTjOO1+EoGnsxB3Adsdi4WjykY1BZ1FpNGW98ajX+lRZG+KIrofizcAxZfhseq28Kr7fNcMPTj2z6GwAA//8DAFBLAwQUAAYACAAAACEAbU1ZqyEGAACOGgAAFQAAAHdvcmQvdGhlbWUvdGhlbWUxLnhtbOxZy47bNhTdF+g/ENo7lm3Jj0E8gS3bSZuZJMg4abOkJVpihhINkpoZIwjQLyhQIC26KdBdC3QToP2D/kuKNv2IUpRlkzbdQToOEBSxAYuPcy8P7yUPJev2nauUgAvEOKZZ32ncch2AspBGOIv7zpPppNZ1ABcwiyChGeo7S8SdO8effnIbHokEpQhI+4wfwb6TCLE4qtd5KJshv0UXKJN9c8pSKGSVxfWIwUvpNyX1puu26ynEmQMymEq30+T3n6Wzh/M5DpFzXHkfE/mTCV40hISdFb7RymSYMwRzhY3OG8WFL3lAGLiApO/IgSJ6OUVXwgEEciE7+o6rPk79+HZ9bUTEHlvNbqI+K7uVQXTeVHYsnq0NPc/32oO1fwUgYhc37ozb4/banwLAMJQzLbnoWH/YG478FVYDlUWL71Fn1GoYeM1/awc/8IuvgVegsujt4CeTYBNDDVQWfUtMOs3AM/AKVBbbO/iOOxh5HQOvQAnB2fkO2vXbraCa7Royp+SeFd7zvUmnuYJvUHVtdZX2mdi31lL4nLKJBKjkQoEzIJYLNIehxAWQ4BnD4ATHiVx4C5hRLpvdpjtxW/K3+HqqpCICjxDUrMumkO80FXwADxleiL7zufTqaJBnObhLRYLD1ai7FvdgFusWb3/65u8fvgJ//frj21ff2vFcx49QFn+JYfZvAwjd4M13r//47fWb77/+85dXFviAwZkOn+IUcfAAXYLHNJWTswyAZuzdLKYJxLrFIIs5zGBhY0GPZfx09IMlJNCCGyIzkk+ZlAob8G7+3CB8lrBcYAvwfpIawFNKyZAy65zuF2PpUciz2D44y3XcYwgvbGMHW3ke5wu55rHNZZAgg+YjIlMOY5QhAYo+eo6QxewZxkZcT3HIKKdzAZ5hMITYGpIpnhmraWN0D6cyL0sbQZlvIzanT8GQEpv7EbowkXJ3QGJziYgRxrswFzC1MoYp0ZEnUCQ2kmdLFhoB50JmOkaEgnGEOLfZPGRLg+59KTH2tJ+SZWoimcDnNuQJpNTY4PQ8SGC6sHLGWaJjP+PncolC8IgKKwlq7pCiLvMgxWNfup9iZKT7+r39RMqQfYEUPTmzbQlEzf24JHOIlPP6lqanOLtW4Lek3X9/0n6KszChds09iKjboTeR8wHD1v20LeL7cNvSHVAW4Q9fuUcwzx4huVks0I/C/VG4//fCvW8/H16uNwqtbuOrm3XlJt175z7HhJyJJUEnXGk7l9OLJrJRVZTR+kFhkcjiajgDFzOoyoBR8QUWyVkCF3KYhhoh5ivXMQcLyuXpoJqtvosOkqenNCpbG43q2VQaQLFpl6dL1S7PIlG2tjubh7C1e1WL1cNyRaCwfRcS2mAmiZaFRKdqvIaEmtlBWPQsLLqF+70s1GWVFbn/ACz+1/C9kpFcb5CgqMhTaV9l9+CZ3hdMc9pNy/R6BdfDZNogoS03k4S2DBMYoe3mA+e6t0mpQa8IxS6NTvd95LoQkS1tIJlZA5dyz7V86SaEi74zl/eFspgupD9e6CYkcdZ3QrEK9H9RlgXjYgR5UsJUVzn/FAvEAMGpXOt6Gki24dZodoo5fqDkeu6HFzl10ZOM5nMUij0tm6rsK51Ye28ILio0l6TPkugSzEjOHkMZKL/TKAIYYS7W0Yww0xb3JopbcrXaisZ/ZpstCskigasTRRfzEq7KazraPBTT7VmZ9dVkZnGRpBufutcbFR2aaO45QIpT064f7++Q11htdN9gVUr3ttb1Kq3bd0rc/EDQqG0GM6gVjC3UNq0mtQPeEGjDrZfmvjPi0KfB9qotDojqvlLVdl5O0NlzufJH8nY1J4IrquhKPiME1d/KpRKo1kpdrgTIGe47L1x/4AVNP6i5XX9c81qeW+v6g1Zt4PutxthvuKNh86UMikjShl+OPZHPM2S5evmi2ndewKTVbfatkKZ1qt6s1JWxegHTaBovYMo3L2Ba9DsAy8i8aDcnvVZv2K71WoNJzRsNu7Ve0B7WRu2gM5qMAr/bm7x0wIUCe4NW4LXH3Vq7EQQ1r+0W9Lu9WsdrNgdeZ9Ade4OXq1jLmVfXKryK1/E/AAAA//8DAFBLAwQKAAAAAAAAACEAvOgH/fQnAAD0JwAAFwAAAGRvY1Byb3BzL3RodW1ibmFpbC5qcGVn/9j/4AAQSkZJRgABAQAASABIAAD/4QCARXhpZgAATU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUAAAABAAAARgEoAAMAAAABAAIAAIdpAAQAAAABAAAATgAAAAAAAABIAAAAAQAAAEgAAAABAAOgAQADAAAAAQABAACgAgAEAAAAAQAAAWmgAwAEAAAAAQAAAgAAAAAA/+0AOFBob3Rvc2hvcCAzLjAAOEJJTQQEAAAAAAAAOEJJTQQlAAAAAAAQ1B2M2Y8AsgTpgAmY7PhCfv/AABEIAgABaQMBEQACEQEDEQH/xAAfAAABBQEBAQEBAQAAAAAAAAAAAQIDBAUGBwgJCgv/xAC1EAACAQMDAgQDBQUEBAAAAX0BAgMABBEFEiExQQYTUWEHInEUMoGRoQgjQrHBFVLR8CQzYnKCCQoWFxgZGiUmJygpKjQ1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4eLj5OXm5+jp6vHy8/T19vf4+fr/xAAfAQADAQEBAQEBAQEBAAAAAAAAAQIDBAUGBwgJCgv/xAC1EQACAQIEBAMEBwUEBAABAncAAQIDEQQFITEGEkFRB2FxEyIygQgUQpGhscEJIzNS8BVictEKFiQ04SXxFxgZGiYnKCkqNTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqCg4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2dri4+Tl5ufo6ery8/T19vf4+fr/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/3QAEAC7/2gAMAwEAAhEDEQA/AP7Yfgx8GPg9N8HvhRLL8KPhrLLL8NfAskkkngTws8kkj+F9LZ3d200s7uxLMzHczEk5JNAHpX/ClPg3/wBEl+GX/hBeFf8A5W0AH/ClPg3/ANEl+GX/AIQXhX/5W0AH/ClPg3/0SX4Zf+EF4V/+VtAB/wAKU+Df/RJfhl/4QXhX/wCVtAB/wpT4N/8ARJfhl/4QXhX/AOVtAB/wpT4N/wDRJfhl/wCEF4V/+VtAB/wpT4N/9El+GX/hBeFf/lbQAf8AClPg3/0SX4Zf+EF4V/8AlbQAf8KU+Df/AESX4Zf+EF4V/wDlbQAf8KU+Df8A0SX4Zf8AhBeFf/lbQAf8KU+Df/RJfhl/4QXhX/5W0AH/AApT4N/9El+GX/hBeFf/AJW0AH/ClPg3/wBEl+GX/hBeFf8A5W0AH/ClPg3/ANEl+GX/AIQXhX/5W0AH/ClPg3/0SX4Zf+EF4V/+VtAB/wAKU+Df/RJfhl/4QXhX/wCVtAB/wpT4N/8ARJfhl/4QXhX/AOVtAB/wpT4N/wDRJfhl/wCEF4V/+VtAB/wpT4N/9El+GX/hBeFf/lbQAf8AClPg3/0SX4Zf+EF4V/8AlbQAf8KU+Df/AESX4Zf+EF4V/wDlbQAf8KU+Df8A0SX4Zf8AhBeFf/lbQAf8KU+Df/RJfhl/4QXhX/5W0AH/AApT4N/9El+GX/hBeFf/AJW0AH/ClPg3/wBEl+GX/hBeFf8A5W0AH/ClPg3/ANEl+GX/AIQXhX/5W0AH/ClPg3/0SX4Zf+EF4V/+VtAB/wAKU+Df/RJfhl/4QXhX/wCVtAB/wpT4N/8ARJfhl/4QXhX/AOVtAB/wpT4N/wDRJfhl/wCEF4V/+VtAB/wpT4N/9El+GX/hBeFf/lbQAf8AClPg3/0SX4Zf+EF4V/8AlbQAf8KU+Df/AESX4Zf+EF4V/wDlbQAf8KU+Df8A0SX4Zf8AhBeFf/lbQAf8KU+Df/RJfhl/4QXhX/5W0AH/AApT4N/9El+GX/hBeFf/AJW0AH/ClPg3/wBEl+GX/hBeFf8A5W0AH/ClPg3/ANEl+GX/AIQXhX/5W0AH/ClPg3/0SX4Zf+EF4V/+VtAB/wAKU+Df/RJfhl/4QXhX/wCVtAB/wpT4N/8ARJfhl/4QXhX/AOVtAB/wpT4N/wDRJfhl/wCEF4V/+VtAB/wpT4N/9El+GX/hBeFf/lbQAf8AClPg3/0SX4Zf+EF4V/8AlbQB/Nd/wrT4c/8ARP8AwT/4Sug//INAH//Q/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9H+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/T/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9T+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAeDfEX4/+Hvhf8QfD3gbxN4W8Vx6ZrfgHxz8TNQ+JS33w9sPh34O8G/DOXRk8fav4uvfEHj3RPE1jD4Xt/Evhu/v307wrq0M1hrUU+ny3Z03Xk0oAxf8Ahrr9n+W68P2WneN7nXLrxJ4x0D4f2sHh7wj411x9L8aeI9T13RrPwx4qGmeHrr/hD9esNV8Na1YeJNG8Uto+q+Eriz2+KLLRxNbvKAXNd/am+C2h+MbHwIPFDa34kn8V6n4Q1Ox8OWU+sSaBf6R8P/iz8Q7+7v4YALrVNPt7T4K+PPCs58JW/ia/tfH2nf8ACHXun22rW2qRaWAZN7+2L+z3bRaLNY+N5/ECa/4p0XwRYyeG/DPinWLW38XeIPh9r/xR07w7rupW+jHSvC2qQ+B/Dt5r2vweJ77SB4Ns7zRZvGjeHoNc0qa6ANXSP2r/ANn7WUia2+JWhxg6X4r1q7mlNxLpWl6R4Asbe7+IWsaj4lsobvwrD4f+H91dQeHvGvimLXZ/C3hzxhJH4O1LWovE8sWkOANvf2s/2eNN06x1bUviZpmnadqdvey6fdahpXiSyS7v9N8cH4a6l4YgW50aGR/HenePlfwnf/D0L/wnNlrUctnc+HonikKgGaP2wfgC/wARfCHwzh8arNrHje68eaVoWsLYXkXhaTxJ8O/Gfw8+Hmu+GJtcuY4Ihqs/jj4m+HPB+l3MME+g3njFbzwS+tW/jP8As/QL8A+nKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoA/mXoA//1f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQB5b8Sfgr8MPi/bahZ/EfwrB4otNV+HPxJ+Emo2t1qGsWltefDz4vW2gWfxF8OTw6ZqNlG0XiO18MaJDJf7Rq2mrZE6Pf6c11etcAHk2k/sUfs26Fq/hbXtJ8DavYaz4JstA07wrqNt8R/ifHc6LY+G/Gcfj/TbS0ZfGKqLZ/FKTX2p28ivBrFpqGs6LqsV3omu65p2oAEGtfsOfsueILvx9fal8MAbj4neIfEXivxo1j4y8f6Qmp+IvF3hL4ieCPE+rW0Ok+KLKDRbjXfDvxY+IUOprocenQXWpeJbrxBJEfEFtp+p2gBQ0H9gr9lfwv8AYz4f+HWraW2nW/hSx06S2+J3xZ86w0/wXbeLLLQ9MtLiTx01xBpf2Dx54107WdMSQWHiTT/FWvWXiKDVLbVLtJQB17+wd+y1qWhWfhXU/h9rOqeEdPvfH9/p3g7VPij8W9S8H6dc/FLRPEnh/wCIT6f4WvvHU2g2K+LNM8Y+K01aO10+FJbrxFrGoIqX17PcMAV9M/YB/ZM0jxR4H8Z2PwtuF8S/Di38NWvgzUZ/iF8T7saNF4R+Is3xY0DFjc+M5tNv5LP4gTvr8s2p2l5JfYTS79rnRkTT6AO9sf2UfgVp97a30PhPWJ5dP8S3vizR4NR+IPxI1XTvD+rah8V/AHxwuYvDek6n4vvNL8OaGPin8MPBPiu28LaJZ2Hhmxk0iXSbHSbfQNX1rS9SAPoqgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKACgAoAKAP5l6AP/W/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9f+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/R/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9L+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/U/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9X+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/X/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9D+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/S/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9P+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/V/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9b+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/Q/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9H+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/T/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9T+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/W/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9f+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/R/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9L+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/U/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9X+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/X/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9D+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/S/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9P+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/V/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9b+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/Q/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9H+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0v7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/T/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9T+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1f7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/W/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9f+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0P7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/R/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9L+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/0/7q/gp/yRr4Sf8AZMvAf/qK6VQB6bQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAfzL0Af/U/ur+Cn/JGvhJ/wBky8B/+orpVAHptABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQB/MvQB/9X+6v4Kf8ka+En/AGTLwH/6iulUAem0AFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFABQAUAFAH8y9AH/1v7Yfgx8Z/g9D8HvhRFL8V/hrFLF8NfAsckcnjvwskkcieF9LV0dG1IMjowKsrDcrAg4INAHpX/C6/g3/wBFa+GX/he+Ff8A5ZUAH/C6/g3/ANFa+GX/AIXvhX/5ZUAH/C6/g3/0Vr4Zf+F74V/+WVAB/wALr+Df/RWvhl/4XvhX/wCWVAB/wuv4N/8ARWvhl/4XvhX/AOWVAB/wuv4N/wDRWvhl/wCF74V/+WVAB/wuv4N/9Fa+GX/he+Ff/llQAf8AC6/g3/0Vr4Zf+F74V/8AllQAf8Lr+Df/AEVr4Zf+F74V/wDllQAf8Lr+Df8A0Vr4Zf8Ahe+Ff/llQAf8Lr+Df/RWvhl/4XvhX/5ZUAH/AAuv4N/9Fa+GX/he+Ff/AJZUAH/C6/g3/wBFa+GX/he+Ff8A5ZUAH/C6/g3/ANFa+GX/AIXvhX/5ZUAH/C6/g3/0Vr4Zf+F74V/+WVAB/wALr+Df/RWvhl/4XvhX/wCWVAB/wuv4N/8ARWvhl/4XvhX/AOWVAB/wuv4N/wDRWvhl/wCF74V/+WVAB/wuv4N/9Fa+GX/he+Ff/llQAf8AC6/g3/0Vr4Zf+F74V/8AllQAf8Lr+Df/AEVr4Zf+F74V/wDllQAf8Lr+Df8A0Vr4Zf8Ahe+Ff/llQAf8Lr+Df/RWvhl/4XvhX/5ZUAH/AAuv4N/9Fa+GX/he+Ff/AJZUAH/C6/g3/wBFa+GX/he+Ff8A5ZUAH/C6/g3/ANFa+GX/AIXvhX/5ZUAH/C6/g3/0Vr4Zf+F74V/+WVAB/wALr+Df/RWvhl/4XvhX/wCWVAB/wuv4N/8ARWvhl/4XvhX/AOWVAB/wuv4N/wDRWvhl/wCF74V/+WVAB/wuv4N/9Fa+GX/he+Ff/llQAf8AC6/g3/0Vr4Zf+F74V/8AllQAf8Lr+Df/AEVr4Zf+F74V/wDllQAf8Lr+Df8A0Vr4Zf8Ahe+Ff/llQAf8Lr+Df/RWvhl/4XvhX/5ZUAH/AAuv4N/9Fa+GX/he+Ff/AJZUAH/C6/g3/wBFa+GX/he+Ff8A5ZUAH/C6/g3/ANFa+GX/AIXvhX/5ZUAH/C6/g3/0Vr4Zf+F74V/+WVAB/wALr+Df/RWvhl/4XvhX/wCWVAB/wuv4N/8ARWvhl/4XvhX/AOWVAB/wuv4N/wDRWvhl/wCF74V/+WVAB/wuv4N/9Fa+GX/he+Ff/llQAf8AC6/g3/0Vr4Zf+F74V/8AllQB/Nd/wsv4c/8ARQPBP/hVaD/8nUAf/9kAAFBLAwQUAAYACAAAACEAuN5y8JsDAACACQAAEQAAAHdvcmQvc2V0dGluZ3MueG1stFZLj9s2EL4X6H8wdK5Wj8iOV403sL1xs8E6WazcS2+URNnE8iEMKatO0f/eESWunGYRuA3ii8n55s1vxn7z9k/BJ0cKmim58KKr0JtQWaiSyf3C+3238efeRBsiS8KVpAvvRLX39ubnn960qabGoJqeoAupU1EsvIMxdRoEujhQQfSVqqlEsFIgiMEr7ANB4Kmp/UKJmhiWM87MKYjDcOYNbtTCa0CmgwtfsAKUVpXpTFJVVaygw5ezgEvi9ia3qmgElcZGDIByzEFJfWC1dt7E//WG4ME5OX6riKPgTq+NwgvKbRWUzxaXpNcZ1KAKqjU+kOAuQSbHwMlXjp5jX2HsoUTrCs2j0J7OM5/+NwfxvxxofkklPXTPciDQ82QoQxTp3V4qIDlHVmI5E8zIu0FaflZKTNq0plDg2yCnw9ALOgA7oqrMEEMR1jXl3JK84JSgwzbdAxFITyexNiWtSMPNjuSZUTUqHQnm/Tqc9/DhVB+otCT6A8fD4Uk87fHiQIAUhkJWkwKjrZU0oLjTK9VHZdY4CoAv1VtocqQPQI+Mtg+sMA3Q3pGdl/GU9bOHjiQR2IAv5mmrStoV1AC7/I06A5tU5HJ/MZDCXQGspLuu8Zk5cbrBmjL2mS5l+aHRhqFH25DvyOBbCWC7MfInpMruVNMNJV2P9A8KZh9ow1m9ZQAK7mSJlPphwVhVUcAADCm6RdYxUK3t83tKSlzR3xk3OKcRLvxSu8OjUsaphuF8Fs5vN32mHXoJslxGr5fJS8jqOpldW0oFz1FF2i3LB3CnjkIT0VusiciBkcm2W6dBp5HD04pJh+cU9wM9R7Imd6Dv94AWhPMNjp4D7AoQacl0fUsre+ZbAvvR76ABL0pxDXx49tWtFQq/gWrqHm2B1D01nEqUJIMlk+aeCSfXTZ45K4kb7QxqZPnpCLZPY3va1OAT2xG7J5YqVrcCf/M4UIlD1tGAbkld92zK99HC42x/MFFHAIO3En917SXfxwMWWyzuMXshRVcZag+HURY72ZneKyd7NcoSJ0tG2dTJpqNs5mSzToZLlALu4icktjt28kpxrlpavh/xr0RuSxcMXzw7iXxcrr/0GGcaJ63GPWwUOOxXi0VJWqriDsmKp/655+t3yTxa9vDU7m+zQx49YWsfabUimpYD5kynvelfm+4zj1f+MrqN/WQ2XfnzeP3OX23iZbReXs+m6/jvYQ7cX6ebfwAAAP//AwBQSwMEFAAGAAgAAAAhAPC8NQHcAQAA8QUAABIAAAB3b3JkL2ZvbnRUYWJsZS54bWy8k9tq4zAQhu8LfQej+8ay4vRg6pQ0bWBh6cXSfQBFkW2xOhhJiTdvvyPZcQMhbJallUHI/4x+jT40j0+/lUx23DphdImyCUYJ18xshK5L9PN9dXOPEuep3lBpNC/Rnjv0NL++euyKymjvEtivXaFYiRrv2yJNHWu4om5iWq4hWBmrqIdfW6eK2l/b9oYZ1VIv1kIKv08JxrdosLGXuJiqEoy/GLZVXPu4P7VcgqPRrhGtO7h1l7h1xm5aaxh3Du6sZO+nqNCjTZafGCnBrHGm8hO4zFBRtILtGY4rJT8MZv9mQEYDxYpvtTaWriXAh0oSMEPzgX7SFZoqCCypFGsrYqCl2jieQWxHZYkwwSs8gzl8OZ6GGaUhkTXUOh5M+kTcyxVVQu4PKt160+ut8Kw5yDtqRaipDzlRQ2Dr1rhErxgGWa1Qr2QlykFYLEeFhKPiyAZlOio4KCz69BkPcReLPmMOnJn2AE5AvAvFXfLGu+SHUVSfAULwLYCYAY4AZvr5QMji9QjIEpS7+/xw/Q8gD38H0mO8HMgCypJnMDwDhnx4GfF1fD6G43cxYJh+BYahQZLvom782TYJzfFFbbIIFZPjVxHahOC75xMc8fL/2SbDws3/AAAA//8DAFBLAwQUAAYACAAAACEA4IvKVR8BAAARAgAAFAAAAHdvcmQvd2ViU2V0dGluZ3MueG1slNFRS8MwEAfwd8HvUPK+pRs6tKwbgkz2MgbVD5Cl1zWY5EIua7dv71nnRHyZbzku9+P+3Hx5dDbrIJJBX4rJOBcZeI218ftSvL2uRg8io6R8rSx6KMUJSCwXtzfzvuhhV0FK/JMyVjwVTpeiTSkUUpJuwSkaYwDPzQajU4nLuJdOxfdDGGl0QSWzM9akk5zm+UycmXiNgk1jNDyjPjjwaZiXESyL6Kk1gb61/hqtx1iHiBqIOI+zX55Txl+Yyd0fyBkdkbBJYw5z3migeHySDy9nf4D7/wHTC+B0sd57jGpn+QS8ScaYWPANlLXYbzcv8rOocYOpUh08UcUpLKyMhaETzBEsbSGuvW6zvuiULcXjTHBT/jrk4gMAAP//AwBQSwMEFAAGAAgAAAAhABZNBGBtAQAA7wIAABEACAFkb2NQcm9wcy9jb3JlLnhtbCCiBAEooAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJySUW+CMBSF35fsP5C+Q4suxhDAZDM+zcRkLlv21rVX7YS2aavIv18BxbH5tLd7e757uJw2nZ3KIjiCsULJDMURQQFIpriQ2wy9rhfhFAXWUclpoSRkqAaLZvn9Xcp0wpSBlVEajBNgA+8kbcJ0hnbO6QRjy3ZQUht5Qnpxo0xJnW/NFmvK9nQLeETIBJfgKKeO4sYw1L0jOlty1lvqgylaA84wFFCCdBbHUYyvrANT2psDrfKDLIWrNdxEL2JPn6zowaqqomrcon7/GL8vn1/aXw2FbLJigPKUs8QJV0Ce4mvpK3v4/ALmuuO+8TUzQJ0y+ZweBQ9WovBdC12EJvI91JUy3PrxQecxDpYZoZ2/yM58cODpglq39De7EcAf61/f+as3IwaOonkZedwSfZueY+52Ax74eJIuzIvyNn6arxcoH5F4EpJxGJM1mSajh4SQj2a9wfzVsDwv8G/Hi0GX0PCJ5t8AAAD//wMAUEsDBBQABgAIAAAAIQCBlv05MgsAAGRyAAAPAAAAd29yZC9zdHlsZXMueG1svJ3bctu6FYbvO9N34OiqvXB8jJ14trPHduLaUzvbO3Kaa4iEJNQgofLgQ5++IEhJkBdBcQGrvrIlan0A8eMHsEBS+u33l1RGTzwvhMrORvsf9kYRz2KViGx2Nvr5cLXzaRQVJcsSJlXGz0avvBj9/uWvf/nt+bQoXyUvIg3IitM0PhvNy3JxurtbxHOesuKDWvBMH5yqPGWlfpnPdlOWP1aLnVilC1aKiZCifN092Ns7HrWYfAhFTaci5l9VXKU8K038bs6lJqqsmItFsaQ9D6E9qzxZ5CrmRaFPOpUNL2UiW2H2jwAoFXGuCjUtP+iTaWtkUDp8f8/8l8o14CMOcLACpPHpzSxTOZtI3fq6JpGGjb7o5k9U/JVPWSXLon6Z3+fty/aV+XOlsrKInk9ZEQvxoEvWkFRo3vV5VoiRPsJZUZ4XgnUenNf/dB6Ji9J6+0IkYrRbl1j8Vx98YvJsdHC0fOeyrsHGe5Jls+V703zn6oddk7MRz3Z+juu3Jpp7NmL5zvi8DtxtT6z5a53uYvWq+dSbttFdQ3eUcdNf9VE+vVXxI0/GpT5wNtqri9Jv/ry5z4XKdZ88G33+3L455qm4FknCM+uD2Vwk/NecZz8Lnqzf//PK9Kv2jVhVmf7/8NOe0UsWybeXmC/qXqqPZqxuve91gKw/XYl14Sb8P0vYfttmXfFzzmqrRvtvEab6KMRBHVFYZ9vNrN6cu/kUqqDD9yro6L0K+vheBR2/V0En71XQp/cqyGD+nwWJLOEvjRFhMYC6jeNwI5rjMBua4/ASmuOwCprjcAKa4+joaI6jH6M5jm6K4JQqdvVCq7MfOnp7P3f7HOHH3T4l+HG3zwB+3O0Dvh93+/jux90+nPtxt4/eftztgzWe2yy1ohtts6wMdtlUqTJTJY9K/hJOY5lmmfyFhldPejwnOUkCTDOytRNxMC1m5vX2HmJM6j+fl3XKFalpNBWzKtdpb2jFefbEpU5AI5YkmkcIzHlZ5Y4W8enTOZ/ynGcxp+zYdFApMh5lVToh6JsLNiNj8Swhbr4lkWRQWHVoVpXz2iSCoFOnLM5VeNUUIxsfbkUR3lY1JLqopORErO80XcywwnMDgwlPDQwmPDMwmPDEwNKMqolaGlFLtTSiBmtpRO3W9E+qdmtpRO3W0ojaraWFt9uDKKUZ4u1Vx/7wvbtLqeod5+B6jMUsY3oBED7dtHum0T3L2Sxni3lU7x93Y+1zxpZzoZLX6IFiTluRqNb1potc6rMWWRXeoBs0KnOteET2WvGIDLbihVvsTi+T6wXaNU0+M64mZadpDWmQacdMVs2CNtxtrAzvYWsDXIm8ILNBN5agB3+vl7O1nBQj37qW4RVbs8Jt9XZUIq1eiySopVTxI80wfP264LlOyx6DSVdKSvXMEzriuMxV09dsyx8YSQZZ/lu6mLNCmFxpAzF8ql9eq47u2CL4hO4lExmNbt92UiZkRLeCuH64u40e1KJOM+uGoQFeqLJUKRmz3Qn82y8++TtNBc91Epy9Ep3tOdH2kIFdCoJJpiGphIikl5kiEyRzqOH9k79OFMsTGtp9zpvbQ0pORByzdNEsOgi8pcfFZz3+EKyGDO9fLBf1vhCVqR5IYNa2YVFN/s3j8KHuu4pIdob+qEqz/2iWuiaaDhe+TNjAhS8RjJp6eqj7L8HJbuDCT3YDR3Wyl5IVhXBeQvXmUZ3ukkd9vuHJX8tTUuXTStI14BJI1oJLIFkTKlmlWUF5xoZHeMKGR32+hF3G8Ai25AzvH7lIyMQwMColDIxKBgOj0sDASAUIv0PHgoXfpmPBwu/VaWBESwALRtXPSKd/oqs8FoyqnxkYVT8zMKp+ZmBU/ezwa8SnU70IpptiLCRVn7OQdBNNVvJ0oXKWvxIhv0k+YwQbpA3tPlfT+rkBlTU3cRMg6z1qSbjYbnBUIv/iE7Kq1SzKehHsiDIplSLaW1tPOCZy8961bWHmmYvgKpjN9lv+xClW4xaM6DJAAwuXzYKFT1MWLHyasmDh05QFC5+mLFj4NGXBwu9fvpcs5nMlE547jNhXkWi8YHF7bQlcox60V38rZvMyGs9Xl6hszPHe1sjlLtNG2PYCuwaK44OesDueiCpdVhQ+AXR8ODzYGHojePmgVk/wevm7EflxYCQs83h75Dq124g8GRgJy/w0MNKMUhuRfYP4V5Y/dnaEk77+s9qYcHS+k75etAruLLavI60iu7rgSV8v2rBKdB7H9SUuqM4wz7jjh5nHHY9xkZuCsZObMthXbkSfwX7wJ1EvRzGDpilvdcvP2+IOzZQ6aOT8s1LNxaaNq6TDn0S80av9rOBRJ+dw+NXWjVHG3Y6Dhxs3YvC440YMHoDciEEjkTMcNSS5KYPHJjdi8CDlRqBHKzgj4EYrGI8brWC8z2gFKT6jVcAqwI0YvBxwI9BGhQi0UQNWCm4Eyqgg3MuokII2KkSgjQoRaKPCBRjOqDAeZ1QY72NUSPExKqSgjQoRaKNCBNqoEIE2KkSgjeq5tneGexkVUtBGhQi0USECbVSzXgwwKozHGRXG+xgVUnyMCiloo0IE2qgQgTYqRKCNChFoo0IEyqgg3MuokII2KkSgjQoRaKM2z8f6GxXG44wK432MCik+RoUUtFEhAm1UiEAbFSLQRoUItFEhAmVUEO5lVEhBGxUi0EaFCLRRzaWDAKPCeJxRYbyPUSHFx6iQgjYqRKCNChFoo0IE2qgQgTYqRKCMCsK9jAopaKNCBNqoENHXP9vr6q5nQ/bxu57Ox0yGX7pqK/XD/v4BG3U4HLWslZs1/AGaC6Ueo86nZQ9NvjEMIiZSKLNF7bgXxOaaC6Soq/V/XPY/lmbTA78prH2Ax1zoB/CjoZFgT+Wor8vbkSDJO+rr6XYkWHUe9Y2+diSYBo/6Bl3jy+WdVHo6AsF9w4wVvO8I7xutrXDYxH1jtBUIW7hvZLYCYQP3jcdW4MeoHpzfRn8c2E7Hq5uiAaGvO1qEEzehr1tCrZbDMTTGUNHchKHquQlDZXQTUHo6MXhh3Si0wm6Un9TQZlip/Y3qJmClhgQvqQHGX2qI8pYaovykhgMjVmpIwErtPzi7CV5SA4y/1BDlLTVE+UkNpzKs1JCAlRoSsFIHTshOjL/UEOUtNUT5SQ0Xd1ipIQErNSRgpYYEL6kBxl9qiPKWGqL8pAZZMlpqSMBKDQlYqSHBS2qA8Zcaorylhqg+qc0uyobUKIWtcNwizArETchWIG5wtgI9siUr2jNbsgie2RLUaqk5LluyRXMThqrnJgyV0U1A6enE4IV1o9AKu1F+UuOypS6p/Y3qJmClxmVLTqlx2VKv1LhsqVdqXLbklhqXLXVJjcuWuqT2H5zdBC+pcdlSr9S4bKlXaly25JYaly11SY3LlrqkxmVLXVIHTshOjL/UuGypV2pctuSWGpctdUmNy5a6pMZlS11S47Ilp9S4bKlXaly21Cs1LltyS43LlrqkxmVLXVLjsqUuqXHZklNqXLbUKzUuW+qV2pEt7T5v/GpYzTa/d6c/XL4ueP3F8dYDM0nzxbntRUDzwZtk9etedXBdk6j9xbP2bVPh9oJhU6IJhEXFc11W3H7ll6OoeyWFPm+WJ/pwCYp0fLOvqcL65JefbhtzfRG0+dzGBc/eGpd1Y/fU1ojBqt72aRRzVfFz2wW31VHXaCKbH8PT/9xkiQY8t7+w1tQ1eWENSh+/5FLesebTauH+qOTTsjm6v2cen31zfNJ8YaEzPjeDhBOwu1mZ5mX7w3eOFm9+wqC9eu1o9fMqrjIutRt4R5ub+ylCm3tdweV/xZf/AQAA//8DAFBLAwQUAAYACAAAACEAQP7QLGkBAAC3AgAAEAAIAWRvY1Byb3BzL2FwcC54bWwgogQBKKAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACcUk1LxTAQvAv+h9K7L32CH8i+iCjiQUV4Vc8h2bbBNAnJKr5/78ZqrXgzp92ZZHZmCZy/j656w5Rt8Jt6vWrqCr0Oxvp+Uz+21wendZVJeaNc8Lipd5jrc7m/Bw8pRExkMVcs4fOmHojimRBZDziqvGLaM9OFNCriNvUidJ3VeBX064iexGHTHAt8J/QGzUGcBetJ8eyN/itqgi7+8lO7i6wnocUxOkUo78tLtzKBRhAzCm0g5Vo7omwYnht4UD1muQYxFfAcksnyEMRUwOWgktLE+5PrExCLFi5idFYr4sXKO6tTyKGj6k5p6ynkoSoKIJa3gENsUb8mS7viY9nCrfWTk6lgZ0n1ScXhy97cwVYrh5ccX3bKZQTxAxSVl/wY23BVYn/xv8FFpmdLwzYqXQafLtMtCNgyioa9zuNmAG54/ckVeX7rezTfd/4SZV9P0z+U66NVw+dzO98YZ5w/iPwAAAD//wMAUEsBAi0AFAAGAAgAAAAhAG2KJ0tmAQAAVAUAABMAAAAAAAAAAAAAAAAAAAAAAFtDb250ZW50X1R5cGVzXS54bWxQSwECLQAUAAYACAAAACEAx8InvP8AAADfAgAACwAAAAAAAAAAAAAAAACfAwAAX3JlbHMvLnJlbHNQSwECLQAUAAYACAAAACEAE6o+h/YAAAAxAwAAHAAAAAAAAAAAAAAAAADPBgAAd29yZC9fcmVscy9kb2N1bWVudC54bWwucmVsc1BLAQItABQABgAIAAAAIQD1Yo5gZQIAAA4HAAARAAAAAAAAAAAAAAAAAAcJAAB3b3JkL2RvY3VtZW50LnhtbFBLAQItABQABgAIAAAAIQBtTVmrIQYAAI4aAAAVAAAAAAAAAAAAAAAAAJsLAAB3b3JkL3RoZW1lL3RoZW1lMS54bWxQSwECLQAKAAAAAAAAACEAvOgH/fQnAAD0JwAAFwAAAAAAAAAAAAAAAADvEQAAZG9jUHJvcHMvdGh1bWJuYWlsLmpwZWdQSwECLQAUAAYACAAAACEAuN5y8JsDAACACQAAEQAAAAAAAAAAAAAAAAAYOgAAd29yZC9zZXR0aW5ncy54bWxQSwECLQAUAAYACAAAACEA8Lw1AdwBAADxBQAAEgAAAAAAAAAAAAAAAADiPQAAd29yZC9mb250VGFibGUueG1sUEsBAi0AFAAGAAgAAAAhAOCLylUfAQAAEQIAABQAAAAAAAAAAAAAAAAA7j8AAHdvcmQvd2ViU2V0dGluZ3MueG1sUEsBAi0AFAAGAAgAAAAhABZNBGBtAQAA7wIAABEAAAAAAAAAAAAAAAAAP0EAAGRvY1Byb3BzL2NvcmUueG1sUEsBAi0AFAAGAAgAAAAhAIGW/TkyCwAAZHIAAA8AAAAAAAAAAAAAAAAA40MAAHdvcmQvc3R5bGVzLnhtbFBLAQItABQABgAIAAAAIQBA/tAsaQEAALcCAAAQAAAAAAAAAAAAAAAAAEJPAABkb2NQcm9wcy9hcHAueG1sUEsFBgAAAAAMAAwABgMAAOFRAAAAAA==" }
+
+  - do:
+      search:
+          index: test
+          body:
+              fields: [file.content, file.author, file.date, file.content_length, file.content_type]
+  - match: { hits.total: 1 }
+  - match: { hits.hits.0.fields: {
+      file.content: ["Test elasticsearch\n"],
+      file.author: ["David Pilato"],
+      file.date: ["2016-03-10T08:24:00Z"],
+      file.content_length: ["21757"],
+      file.content_type: ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]
+    }
+  }
+
diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperUpgradeTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperUpgradeTests.java
index fe12cb042d4..c632e139955 100644
--- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperUpgradeTests.java
+++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperUpgradeTests.java
@@ -52,6 +52,7 @@ public class Murmur3FieldMapperUpgradeTests extends ESIntegTestCase {
 
     public void testUpgradeOldMapping() throws IOException, ExecutionException, InterruptedException {
         final String indexName = "index-mapper-murmur3-2.0.0";
+        final String indexUUID = "1VzJds59TTK7lRu17W0mcg";
         InternalTestCluster.Async<String> master = internalCluster().startNodeAsync();
         Path unzipDir = createTempDir();
         Path unzipDataDir = unzipDir.resolve("data");
@@ -72,6 +73,7 @@ public class Murmur3FieldMapperUpgradeTests extends ESIntegTestCase {
         assertFalse(Files.exists(dataPath));
         Path src = unzipDataDir.resolve(indexName + "/nodes/0/indices");
         Files.move(src, dataPath);
+        Files.move(dataPath.resolve(indexName), dataPath.resolve(indexUUID));
 
         master.get();
         // force reloading dangling indices with a cluster state republish
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
index a2af6df4e75..761fb5fd144 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
@@ -53,6 +53,7 @@ public class SizeFieldMapperUpgradeTests extends ESIntegTestCase {
 
     public void testUpgradeOldMapping() throws IOException, ExecutionException, InterruptedException {
         final String indexName = "index-mapper-size-2.0.0";
+        final String indexUUID = "ENCw7sG0SWuTPcH60bHheg";
         InternalTestCluster.Async<String> master = internalCluster().startNodeAsync();
         Path unzipDir = createTempDir();
         Path unzipDataDir = unzipDir.resolve("data");
@@ -73,6 +74,7 @@ public class SizeFieldMapperUpgradeTests extends ESIntegTestCase {
         assertFalse(Files.exists(dataPath));
         Path src = unzipDataDir.resolve(indexName + "/nodes/0/indices");
         Files.move(src, dataPath);
+        Files.move(dataPath.resolve(indexName), dataPath.resolve(indexUUID));
         master.get();
         // force reloading dangling indices with a cluster state republish
         client().admin().cluster().prepareReroute().get();
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
index 1c3ece1adad..01d66c177a2 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
@@ -24,6 +24,7 @@ import com.microsoft.azure.storage.StorageException;
 
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 
@@ -31,7 +32,6 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URISyntaxException;
 import java.util.Map;
-import java.util.function.Function;
 
 /**
  * Azure Storage Service interface
@@ -41,13 +41,20 @@ public interface AzureStorageService {
 
     final class Storage {
         public static final String PREFIX = "cloud.azure.storage.";
-        public static final Setting<TimeValue> TIMEOUT_SETTING = Setting.timeSetting("cloud.azure.storage.timeout", TimeValue.timeValueSeconds(-1), false, Setting.Scope.CLUSTER);
-        public static final Setting<String> ACCOUNT_SETTING = Setting.simpleString("repositories.azure.account", false, Setting.Scope.CLUSTER);
-        public static final Setting<String> CONTAINER_SETTING = Setting.simpleString("repositories.azure.container", false, Setting.Scope.CLUSTER);
-        public static final Setting<String> BASE_PATH_SETTING = Setting.simpleString("repositories.azure.base_path", false, Setting.Scope.CLUSTER);
-        public static final Setting<String> LOCATION_MODE_SETTING = Setting.simpleString("repositories.azure.location_mode", false, Setting.Scope.CLUSTER);
-        public static final Setting<ByteSizeValue> CHUNK_SIZE_SETTING = Setting.byteSizeSetting("repositories.azure.chunk_size", new ByteSizeValue(-1), false, Setting.Scope.CLUSTER);
-        public static final Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("repositories.azure.compress", false, false, Setting.Scope.CLUSTER);
+        public static final Setting<TimeValue> TIMEOUT_SETTING =
+            Setting.timeSetting("cloud.azure.storage.timeout", TimeValue.timeValueMinutes(-1), Property.NodeScope);
+        public static final Setting<String> ACCOUNT_SETTING =
+            Setting.simpleString("repositories.azure.account", Property.NodeScope, Property.Filtered);
+        public static final Setting<String> CONTAINER_SETTING =
+            Setting.simpleString("repositories.azure.container", Property.NodeScope);
+        public static final Setting<String> BASE_PATH_SETTING =
+            Setting.simpleString("repositories.azure.base_path", Property.NodeScope);
+        public static final Setting<String> LOCATION_MODE_SETTING =
+            Setting.simpleString("repositories.azure.location_mode", Property.NodeScope);
+        public static final Setting<ByteSizeValue> CHUNK_SIZE_SETTING =
+            Setting.byteSizeSetting("repositories.azure.chunk_size", new ByteSizeValue(-1), Property.NodeScope);
+        public static final Setting<Boolean> COMPRESS_SETTING =
+            Setting.boolSetting("repositories.azure.compress", false, Property.NodeScope);
     }
 
     boolean doesContainerExist(String account, LocationMode mode, String container);
@@ -62,13 +69,17 @@ public interface AzureStorageService {
 
     void deleteBlob(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException;
 
-    InputStream getInputStream(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException;
+    InputStream getInputStream(String account, LocationMode mode, String container, String blob)
+        throws URISyntaxException, StorageException;
 
-    OutputStream getOutputStream(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException;
+    OutputStream getOutputStream(String account, LocationMode mode, String container, String blob)
+        throws URISyntaxException, StorageException;
 
-    Map<String,BlobMetaData> listBlobsByPrefix(String account, LocationMode mode, String container, String keyPath, String prefix) throws URISyntaxException, StorageException;
+    Map<String,BlobMetaData> listBlobsByPrefix(String account, LocationMode mode, String container, String keyPath, String prefix)
+        throws URISyntaxException, StorageException;
 
-    void moveBlob(String account, LocationMode mode, String container, String sourceBlob, String targetBlob) throws URISyntaxException, StorageException;
+    void moveBlob(String account, LocationMode mode, String container, String sourceBlob, String targetBlob)
+        throws URISyntaxException, StorageException;
 
     AzureStorageService start();
 }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
index cdc6d74edb0..497b0e3753a 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
@@ -169,7 +169,7 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
             logger.trace("creating container [{}]", container);
             blob_container.createIfNotExists();
         } catch (IllegalArgumentException e) {
-            logger.trace("fails creating container [{}]", container, e.getMessage());
+            logger.trace("fails creating container [{}]", e, container);
             throw new RepositoryException(container, e.getMessage());
         }
     }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
index da9151e9504..281ef79cb27 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
@@ -46,11 +46,13 @@ public final class AzureStorageSettings {
         TIMEOUT_KEY,
         (s) -> Storage.TIMEOUT_SETTING.get(s).toString(),
         (s) -> Setting.parseTimeValue(s, TimeValue.timeValueSeconds(-1), TIMEOUT_KEY.toString()),
-        false,
-        Setting.Scope.CLUSTER);
-    private static final Setting<String> ACCOUNT_SETTING = Setting.adfixKeySetting(Storage.PREFIX, ACCOUNT_SUFFIX, "", Function.identity(), false, Setting.Scope.CLUSTER);
-    private static final Setting<String> KEY_SETTING = Setting.adfixKeySetting(Storage.PREFIX, KEY_SUFFIX, "", Function.identity(), false, Setting.Scope.CLUSTER);
-    private static final Setting<Boolean> DEFAULT_SETTING = Setting.adfixKeySetting(Storage.PREFIX, DEFAULT_SUFFIX, "false", Boolean::valueOf, false, Setting.Scope.CLUSTER);
+        Setting.Property.NodeScope);
+    private static final Setting<String> ACCOUNT_SETTING =
+        Setting.adfixKeySetting(Storage.PREFIX, ACCOUNT_SUFFIX, "", Function.identity(), Setting.Property.NodeScope);
+    private static final Setting<String> KEY_SETTING =
+        Setting.adfixKeySetting(Storage.PREFIX, KEY_SUFFIX, "", Function.identity(), Setting.Property.NodeScope);
+    private static final Setting<Boolean> DEFAULT_SETTING =
+        Setting.adfixKeySetting(Storage.PREFIX, DEFAULT_SUFFIX, "false", Boolean::valueOf, Setting.Property.NodeScope);
 
 
     private final String name;
@@ -110,7 +112,7 @@ public final class AzureStorageSettings {
     }
 
     private static List<AzureStorageSettings> createStorageSettings(Settings settings) {
-        Setting<Settings> storageGroupSetting = Setting.groupSetting(Storage.PREFIX, false, Setting.Scope.CLUSTER);
+        Setting<Settings> storageGroupSetting = Setting.groupSetting(Storage.PREFIX, Setting.Property.NodeScope);
         // ignore global timeout which has the same prefix but does not belong to any group
         Settings groups = storageGroupSetting.get(settings.filter((k) -> k.equals(Storage.TIMEOUT_SETTING.getKey()) == false));
         List<AzureStorageSettings> storageSettings = new ArrayList<>();
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
index 616b150f954..3ce043500ae 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
@@ -74,9 +74,9 @@ public class AzureRepositoryPlugin extends Plugin {
         module.registerSetting(AzureStorageService.Storage.BASE_PATH_SETTING);
         module.registerSetting(AzureStorageService.Storage.CHUNK_SIZE_SETTING);
         module.registerSetting(AzureStorageService.Storage.LOCATION_MODE_SETTING);
-        // Cloud storage API settings needed to be hidden
+
+        // Cloud storage API settings using a pattern needed to be hidden
         module.registerSettingsFilter(AzureStorageService.Storage.PREFIX + "*.account");
         module.registerSettingsFilter(AzureStorageService.Storage.PREFIX + "*.key");
-        module.registerSettingsFilter(AzureStorageService.Storage.ACCOUNT_SETTING.getKey());
     }
 }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
index f2773bccbbd..66db57fdd92 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -67,12 +68,14 @@ public class AzureRepository extends BlobStoreRepository {
     public final static String TYPE = "azure";
 
     public static final class Repository {
-        public static final Setting<String> ACCOUNT_SETTING = Setting.simpleString("account", false, Setting.Scope.CLUSTER);
-        public static final Setting<String> CONTAINER_SETTING = new Setting<>("container", "elasticsearch-snapshots", Function.identity(), false, Setting.Scope.CLUSTER);
-        public static final Setting<String> BASE_PATH_SETTING = Setting.simpleString("base_path", false, Setting.Scope.CLUSTER);
-        public static final Setting<String> LOCATION_MODE_SETTING = Setting.simpleString("location_mode", false, Setting.Scope.CLUSTER);
-        public static final Setting<ByteSizeValue> CHUNK_SIZE_SETTING = Setting.byteSizeSetting("chunk_size", MAX_CHUNK_SIZE, false, Setting.Scope.CLUSTER);
-        public static final Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("compress", false, false, Setting.Scope.CLUSTER);
+        public static final Setting<String> ACCOUNT_SETTING = Setting.simpleString("account", Property.NodeScope);
+        public static final Setting<String> CONTAINER_SETTING =
+            new Setting<>("container", "elasticsearch-snapshots", Function.identity(), Property.NodeScope);
+        public static final Setting<String> BASE_PATH_SETTING = Setting.simpleString("base_path", Property.NodeScope);
+        public static final Setting<String> LOCATION_MODE_SETTING = Setting.simpleString("location_mode", Property.NodeScope);
+        public static final Setting<ByteSizeValue> CHUNK_SIZE_SETTING =
+            Setting.byteSizeSetting("chunk_size", MAX_CHUNK_SIZE, Property.NodeScope);
+        public static final Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("compress", false, Property.NodeScope);
     }
 
     private final AzureBlobStore blobStore;
diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index 915a85ebdc4..8fc9e50d7f3 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -45,6 +45,7 @@ dependencies {
   compile 'com.google.guava:guava:16.0.1'
   compile 'com.google.protobuf:protobuf-java:2.5.0'
   compile 'commons-logging:commons-logging:1.1.3'
+  compile 'commons-cli:commons-cli:1.2'
   compile 'commons-collections:commons-collections:3.2.2'
   compile 'commons-configuration:commons-configuration:1.6'
   compile 'commons-io:commons-io:2.4'
diff --git a/plugins/repository-hdfs/licenses/commons-cli-1.2.jar.sha1 b/plugins/repository-hdfs/licenses/commons-cli-1.2.jar.sha1
new file mode 100644
index 00000000000..d38d00127e8
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/commons-cli-1.2.jar.sha1
@@ -0,0 +1 @@
+2bf96b7aa8b611c177d329452af1dc933e14501c
\ No newline at end of file
diff --git a/distribution/licenses/commons-cli-LICENSE.txt b/plugins/repository-hdfs/licenses/commons-cli-LICENSE.txt
similarity index 100%
rename from distribution/licenses/commons-cli-LICENSE.txt
rename to plugins/repository-hdfs/licenses/commons-cli-LICENSE.txt
diff --git a/distribution/licenses/commons-cli-NOTICE.txt b/plugins/repository-hdfs/licenses/commons-cli-NOTICE.txt
similarity index 100%
rename from distribution/licenses/commons-cli-NOTICE.txt
rename to plugins/repository-hdfs/licenses/commons-cli-NOTICE.txt
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
index 3ccd6d7987f..427c454fa28 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
@@ -23,6 +23,7 @@ import com.amazonaws.Protocol;
 import com.amazonaws.services.s3.AmazonS3;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 
 import java.util.Locale;
 import java.util.function.Function;
@@ -38,40 +39,44 @@ public interface AwsS3Service extends LifecycleComponent<AwsS3Service> {
     /**
      * cloud.aws.access_key: AWS Access key. Shared with discovery-ec2 plugin
      */
-    Setting<String> KEY_SETTING = Setting.simpleString("cloud.aws.access_key", false, Setting.Scope.CLUSTER);
+    Setting<String> KEY_SETTING =
+        Setting.simpleString("cloud.aws.access_key", Property.NodeScope, Property.Filtered);
     /**
      * cloud.aws.secret_key: AWS Secret key. Shared with discovery-ec2 plugin
      */
-    Setting<String> SECRET_SETTING = Setting.simpleString("cloud.aws.secret_key", false, Setting.Scope.CLUSTER);
+    Setting<String> SECRET_SETTING =
+        Setting.simpleString("cloud.aws.secret_key", Property.NodeScope, Property.Filtered);
     /**
      * cloud.aws.protocol: Protocol for AWS API: http or https. Defaults to https. Shared with discovery-ec2 plugin
      */
     Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
-        false, Setting.Scope.CLUSTER);
+        Property.NodeScope);
     /**
      * cloud.aws.proxy.host: In case of proxy, define its hostname/IP. Shared with discovery-ec2 plugin
      */
-    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host", false, Setting.Scope.CLUSTER);
+    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host", Property.NodeScope);
     /**
      * cloud.aws.proxy.port: In case of proxy, define its port. Defaults to 80. Shared with discovery-ec2 plugin
      */
-    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16, false, Setting.Scope.CLUSTER);
+    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16, Property.NodeScope);
     /**
      * cloud.aws.proxy.username: In case of proxy with auth, define the username. Shared with discovery-ec2 plugin
      */
-    Setting<String> PROXY_USERNAME_SETTING = Setting.simpleString("cloud.aws.proxy.username", false, Setting.Scope.CLUSTER);
+    Setting<String> PROXY_USERNAME_SETTING = Setting.simpleString("cloud.aws.proxy.username", Property.NodeScope);
     /**
      * cloud.aws.proxy.password: In case of proxy with auth, define the password. Shared with discovery-ec2 plugin
      */
-    Setting<String> PROXY_PASSWORD_SETTING = Setting.simpleString("cloud.aws.proxy.password", false, Setting.Scope.CLUSTER);
+    Setting<String> PROXY_PASSWORD_SETTING =
+        Setting.simpleString("cloud.aws.proxy.password", Property.NodeScope, Property.Filtered);
     /**
      * cloud.aws.signer: If you are using an old AWS API version, you can define a Signer. Shared with discovery-ec2 plugin
      */
-    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", false, Setting.Scope.CLUSTER);
+    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", Property.NodeScope);
     /**
      * cloud.aws.region: Region. Shared with discovery-ec2 plugin
      */
-    Setting<String> REGION_SETTING = new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), false, Setting.Scope.CLUSTER);
+    Setting<String> REGION_SETTING =
+        new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
 
     /**
      * Defines specific s3 settings starting with cloud.aws.s3.
@@ -82,68 +87,70 @@ public interface AwsS3Service extends LifecycleComponent<AwsS3Service> {
          * @see AwsS3Service#KEY_SETTING
          */
         Setting<String> KEY_SETTING =
-            new Setting<>("cloud.aws.s3.access_key", AwsS3Service.KEY_SETTING, Function.identity(), false, Setting.Scope.CLUSTER);
+            new Setting<>("cloud.aws.s3.access_key", AwsS3Service.KEY_SETTING, Function.identity(),
+                Property.NodeScope, Property.Filtered);
         /**
          * cloud.aws.s3.secret_key: AWS Secret key specific for S3 API calls. Defaults to cloud.aws.secret_key.
          * @see AwsS3Service#SECRET_SETTING
          */
         Setting<String> SECRET_SETTING =
-            new Setting<>("cloud.aws.s3.secret_key", AwsS3Service.SECRET_SETTING, Function.identity(), false, Setting.Scope.CLUSTER);
+            new Setting<>("cloud.aws.s3.secret_key", AwsS3Service.SECRET_SETTING, Function.identity(),
+                Property.NodeScope, Property.Filtered);
         /**
          * cloud.aws.s3.protocol: Protocol for AWS API specific for S3 API calls: http or https. Defaults to cloud.aws.protocol.
          * @see AwsS3Service#PROTOCOL_SETTING
          */
         Setting<Protocol> PROTOCOL_SETTING =
-            new Setting<>("cloud.aws.s3.protocol", AwsS3Service.PROTOCOL_SETTING, s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), false,
-                Setting.Scope.CLUSTER);
+            new Setting<>("cloud.aws.s3.protocol", AwsS3Service.PROTOCOL_SETTING, s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
+                Property.NodeScope);
         /**
          * cloud.aws.s3.proxy.host: In case of proxy, define its hostname/IP specific for S3 API calls. Defaults to cloud.aws.proxy.host.
          * @see AwsS3Service#PROXY_HOST_SETTING
          */
         Setting<String> PROXY_HOST_SETTING =
-            new Setting<>("cloud.aws.s3.proxy.host", AwsS3Service.PROXY_HOST_SETTING, Function.identity(), false, Setting.Scope.CLUSTER);
+            new Setting<>("cloud.aws.s3.proxy.host", AwsS3Service.PROXY_HOST_SETTING, Function.identity(),
+                Property.NodeScope);
         /**
          * cloud.aws.s3.proxy.port: In case of proxy, define its port specific for S3 API calls.  Defaults to cloud.aws.proxy.port.
          * @see AwsS3Service#PROXY_PORT_SETTING
          */
         Setting<Integer> PROXY_PORT_SETTING =
             new Setting<>("cloud.aws.s3.proxy.port", AwsS3Service.PROXY_PORT_SETTING,
-                s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.s3.proxy.port"), false, Setting.Scope.CLUSTER);
+                s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.s3.proxy.port"), Property.NodeScope);
         /**
          * cloud.aws.s3.proxy.username: In case of proxy with auth, define the username specific for S3 API calls.
          * Defaults to cloud.aws.proxy.username.
          * @see AwsS3Service#PROXY_USERNAME_SETTING
          */
         Setting<String> PROXY_USERNAME_SETTING =
-            new Setting<>("cloud.aws.s3.proxy.username", AwsS3Service.PROXY_USERNAME_SETTING, Function.identity(), false,
-                Setting.Scope.CLUSTER);
+            new Setting<>("cloud.aws.s3.proxy.username", AwsS3Service.PROXY_USERNAME_SETTING, Function.identity(),
+                Property.NodeScope);
         /**
          * cloud.aws.s3.proxy.password: In case of proxy with auth, define the password specific for S3 API calls.
          * Defaults to cloud.aws.proxy.password.
          * @see AwsS3Service#PROXY_PASSWORD_SETTING
          */
         Setting<String> PROXY_PASSWORD_SETTING =
-            new Setting<>("cloud.aws.s3.proxy.password", AwsS3Service.PROXY_PASSWORD_SETTING, Function.identity(), false,
-                Setting.Scope.CLUSTER);
+            new Setting<>("cloud.aws.s3.proxy.password", AwsS3Service.PROXY_PASSWORD_SETTING, Function.identity(),
+                Property.NodeScope, Property.Filtered);
         /**
          * cloud.aws.s3.signer: If you are using an old AWS API version, you can define a Signer. Specific for S3 API calls.
          * Defaults to cloud.aws.signer.
          * @see AwsS3Service#SIGNER_SETTING
          */
         Setting<String> SIGNER_SETTING =
-            new Setting<>("cloud.aws.s3.signer", AwsS3Service.SIGNER_SETTING, Function.identity(), false, Setting.Scope.CLUSTER);
+            new Setting<>("cloud.aws.s3.signer", AwsS3Service.SIGNER_SETTING, Function.identity(), Property.NodeScope);
         /**
          * cloud.aws.s3.region: Region specific for S3 API calls. Defaults to cloud.aws.region.
          * @see AwsS3Service#REGION_SETTING
          */
         Setting<String> REGION_SETTING =
-            new Setting<>("cloud.aws.s3.region", AwsS3Service.REGION_SETTING, s -> s.toLowerCase(Locale.ROOT), false,
-                Setting.Scope.CLUSTER);
+            new Setting<>("cloud.aws.s3.region", AwsS3Service.REGION_SETTING, s -> s.toLowerCase(Locale.ROOT),
+                Property.NodeScope);
         /**
          * cloud.aws.s3.endpoint: Endpoint. If not set, endpoint will be guessed based on region setting.
          */
-        Setting<String> ENDPOINT_SETTING =
-            Setting.simpleString("cloud.aws.s3.endpoint", false, Setting.Scope.CLUSTER);
+        Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.s3.endpoint", Property.NodeScope);
     }
 
     AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries);
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java
index c94491696c0..5c02671e5e9 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsSigner.java
@@ -64,7 +64,7 @@ public class AwsSigner {
         try {
             validateSignerType(signer, endpoint);
         } catch (IllegalArgumentException e) {
-            logger.warn(e.getMessage());
+            logger.warn("{}", e.getMessage());
         }
 
         configuration.setSignerOverride(signer);
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java
index a8bf3ea2959..650d71f62ad 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java
@@ -36,7 +36,6 @@ import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.blobstore.BlobStoreException;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 
 import java.util.ArrayList;
@@ -47,8 +46,6 @@ import java.util.Locale;
  */
 public class S3BlobStore extends AbstractComponent implements BlobStore {
 
-    public static final ByteSizeValue MIN_BUFFER_SIZE = new ByteSizeValue(5, ByteSizeUnit.MB);
-
     private final AmazonS3 client;
 
     private final String bucket;
@@ -72,12 +69,7 @@ public class S3BlobStore extends AbstractComponent implements BlobStore {
         this.bucket = bucket;
         this.region = region;
         this.serverSideEncryption = serverSideEncryption;
-
-        this.bufferSize = (bufferSize != null) ? bufferSize : MIN_BUFFER_SIZE;
-        if (this.bufferSize.getBytes() < MIN_BUFFER_SIZE.getBytes()) {
-            throw new BlobStoreException("Detected a buffer_size for the S3 storage lower than [" + MIN_BUFFER_SIZE + "]");
-        }
-
+        this.bufferSize = bufferSize;
         this.cannedACL = initCannedACL(cannedACL);
         this.numberOfRetries = maxRetries;
         this.storageClass = initStorageClass(storageClass);
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
index 5d21bb4e2ac..d07d8c174c5 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
@@ -144,16 +144,6 @@ public class S3RepositoryPlugin extends Plugin {
         settingsModule.registerSetting(S3Repository.Repository.STORAGE_CLASS_SETTING);
         settingsModule.registerSetting(S3Repository.Repository.CANNED_ACL_SETTING);
         settingsModule.registerSetting(S3Repository.Repository.BASE_PATH_SETTING);
-
-        // Filter global settings
-        settingsModule.registerSettingsFilterIfMissing(AwsS3Service.KEY_SETTING.getKey());
-        settingsModule.registerSettingsFilterIfMissing(AwsS3Service.SECRET_SETTING.getKey());
-        settingsModule.registerSettingsFilterIfMissing(AwsS3Service.PROXY_PASSWORD_SETTING.getKey());
-        settingsModule.registerSettingsFilterIfMissing(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey());
-        settingsModule.registerSettingsFilterIfMissing(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey());
-        settingsModule.registerSettingsFilterIfMissing(AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.getKey());
-        settingsModule.registerSettingsFilter(S3Repository.Repository.KEY_SETTING.getKey());
-        settingsModule.registerSettingsFilter(S3Repository.Repository.SECRET_SETTING.getKey());
     }
 
     /**
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
index 3edead0765e..a09d57ebc93 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -28,6 +28,7 @@ import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.snapshots.IndexShardRepository;
@@ -65,70 +66,80 @@ public class S3Repository extends BlobStoreRepository {
          * repositories.s3.access_key: AWS Access key specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.access_key.
          * @see CLOUD_S3#KEY_SETTING
          */
-        Setting<String> KEY_SETTING = new Setting<>("repositories.s3.access_key", CLOUD_S3.KEY_SETTING, Function.identity(), false, Setting.Scope.CLUSTER);
+        Setting<String> KEY_SETTING =
+            new Setting<>("repositories.s3.access_key", CLOUD_S3.KEY_SETTING, Function.identity(), Property.NodeScope);
         /**
          * repositories.s3.secret_key: AWS Secret key specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.secret_key.
          * @see CLOUD_S3#SECRET_SETTING
          */
-        Setting<String> SECRET_SETTING = new Setting<>("repositories.s3.secret_key", CLOUD_S3.SECRET_SETTING, Function.identity(), false, Setting.Scope.CLUSTER);
+        Setting<String> SECRET_SETTING =
+            new Setting<>("repositories.s3.secret_key", CLOUD_S3.SECRET_SETTING, Function.identity(), Property.NodeScope);
         /**
          * repositories.s3.region: Region specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.region.
          * @see CLOUD_S3#REGION_SETTING
          */
-        Setting<String> REGION_SETTING = new Setting<>("repositories.s3.region", CLOUD_S3.REGION_SETTING, s -> s.toLowerCase(Locale.ROOT), false, Setting.Scope.CLUSTER);
+        Setting<String> REGION_SETTING =
+            new Setting<>("repositories.s3.region", CLOUD_S3.REGION_SETTING, s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
         /**
          * repositories.s3.endpoint: Endpoint specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.endpoint.
          * @see CLOUD_S3#ENDPOINT_SETTING
          */
-        Setting<String> ENDPOINT_SETTING = new Setting<>("repositories.s3.endpoint", CLOUD_S3.ENDPOINT_SETTING, s -> s.toLowerCase(Locale.ROOT), false, Setting.Scope.CLUSTER);
+        Setting<String> ENDPOINT_SETTING =
+            new Setting<>("repositories.s3.endpoint", CLOUD_S3.ENDPOINT_SETTING, s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
         /**
          * repositories.s3.protocol: Protocol specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.protocol.
          * @see CLOUD_S3#PROTOCOL_SETTING
          */
-        Setting<Protocol> PROTOCOL_SETTING = new Setting<>("repositories.s3.protocol", CLOUD_S3.PROTOCOL_SETTING, s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), false, Setting.Scope.CLUSTER);
+        Setting<Protocol> PROTOCOL_SETTING =
+            new Setting<>("repositories.s3.protocol", CLOUD_S3.PROTOCOL_SETTING, s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope);
         /**
          * repositories.s3.bucket: The name of the bucket to be used for snapshots.
          */
-        Setting<String> BUCKET_SETTING = Setting.simpleString("repositories.s3.bucket", false, Setting.Scope.CLUSTER);
+        Setting<String> BUCKET_SETTING = Setting.simpleString("repositories.s3.bucket", Property.NodeScope);
         /**
          * repositories.s3.server_side_encryption: When set to true files are encrypted on server side using AES256 algorithm.
          * Defaults to false.
          */
-        Setting<Boolean> SERVER_SIDE_ENCRYPTION_SETTING = Setting.boolSetting("repositories.s3.server_side_encryption", false, false, Setting.Scope.CLUSTER);
+        Setting<Boolean> SERVER_SIDE_ENCRYPTION_SETTING =
+            Setting.boolSetting("repositories.s3.server_side_encryption", false, Property.NodeScope);
         /**
          * repositories.s3.buffer_size: Minimum threshold below which the chunk is uploaded using a single request. Beyond this threshold,
          * the S3 repository will use the AWS Multipart Upload API to split the chunk into several parts, each of buffer_size length, and
          * to upload each part in its own request. Note that setting a buffer size lower than 5mb is not allowed since it will prevents the
-         * use of the Multipart API and may result in upload errors. Defaults to 5mb.
+         * use of the Multipart API and may result in upload errors. Defaults to 100m.
          */
-        Setting<ByteSizeValue> BUFFER_SIZE_SETTING = Setting.byteSizeSetting("repositories.s3.buffer_size", S3BlobStore.MIN_BUFFER_SIZE, false, Setting.Scope.CLUSTER);
+        Setting<ByteSizeValue> BUFFER_SIZE_SETTING =
+            Setting.byteSizeSetting("repositories.s3.buffer_size", new ByteSizeValue(100, ByteSizeUnit.MB),
+                new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope);
         /**
          * repositories.s3.max_retries: Number of retries in case of S3 errors. Defaults to 3.
          */
-        Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("repositories.s3.max_retries", 3, false, Setting.Scope.CLUSTER);
+        Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("repositories.s3.max_retries", 3, Property.NodeScope);
         /**
-         * repositories.s3.chunk_size: Big files can be broken down into chunks during snapshotting if needed. Defaults to 100m.
+         * repositories.s3.chunk_size: Big files can be broken down into chunks during snapshotting if needed. Defaults to 1g.
          */
-        Setting<ByteSizeValue> CHUNK_SIZE_SETTING = Setting.byteSizeSetting("repositories.s3.chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB), false, Setting.Scope.CLUSTER);
+        Setting<ByteSizeValue> CHUNK_SIZE_SETTING =
+            Setting.byteSizeSetting("repositories.s3.chunk_size", new ByteSizeValue(1, ByteSizeUnit.GB),
+                new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope);
         /**
          * repositories.s3.compress: When set to true metadata files are stored in compressed format. This setting doesn’t affect index
          * files that are already compressed by default. Defaults to false.
          */
-        Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("repositories.s3.compress", false, false, Setting.Scope.CLUSTER);
+        Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("repositories.s3.compress", false, Property.NodeScope);
         /**
          * repositories.s3.storage_class: Sets the S3 storage class type for the backup files. Values may be standard, reduced_redundancy,
          * standard_ia. Defaults to standard.
          */
-        Setting<String> STORAGE_CLASS_SETTING = Setting.simpleString("repositories.s3.storage_class", false, Setting.Scope.CLUSTER);
+        Setting<String> STORAGE_CLASS_SETTING = Setting.simpleString("repositories.s3.storage_class", Property.NodeScope);
         /**
          * repositories.s3.canned_acl: The S3 repository supports all S3 canned ACLs : private, public-read, public-read-write,
          * authenticated-read, log-delivery-write, bucket-owner-read, bucket-owner-full-control. Defaults to private.
          */
-        Setting<String> CANNED_ACL_SETTING = Setting.simpleString("repositories.s3.canned_acl", false, Setting.Scope.CLUSTER);
+        Setting<String> CANNED_ACL_SETTING = Setting.simpleString("repositories.s3.canned_acl", Property.NodeScope);
         /**
          * repositories.s3.base_path: Specifies the path within bucket to repository data. Defaults to root directory.
          */
-        Setting<String> BASE_PATH_SETTING = Setting.simpleString("repositories.s3.base_path", false, Setting.Scope.CLUSTER);
+        Setting<String> BASE_PATH_SETTING = Setting.simpleString("repositories.s3.base_path", Property.NodeScope);
     }
 
     /**
@@ -140,72 +151,78 @@ public class S3Repository extends BlobStoreRepository {
          * access_key
          * @see  Repositories#KEY_SETTING
          */
-        Setting<String> KEY_SETTING = Setting.simpleString("access_key", false, Setting.Scope.CLUSTER);
+        Setting<String> KEY_SETTING = Setting.simpleString("access_key", Property.NodeScope, Property.Filtered);
         /**
          * secret_key
          * @see  Repositories#SECRET_SETTING
          */
-        Setting<String> SECRET_SETTING = Setting.simpleString("secret_key", false, Setting.Scope.CLUSTER);
+        Setting<String> SECRET_SETTING = Setting.simpleString("secret_key", Property.NodeScope, Property.Filtered);
         /**
          * bucket
          * @see  Repositories#BUCKET_SETTING
          */
-        Setting<String> BUCKET_SETTING = Setting.simpleString("bucket", false, Setting.Scope.CLUSTER);
+        Setting<String> BUCKET_SETTING = Setting.simpleString("bucket", Property.NodeScope);
         /**
          * endpoint
          * @see  Repositories#ENDPOINT_SETTING
          */
-        Setting<String> ENDPOINT_SETTING = Setting.simpleString("endpoint", false, Setting.Scope.CLUSTER);
+        Setting<String> ENDPOINT_SETTING = Setting.simpleString("endpoint", Property.NodeScope);
         /**
          * protocol
          * @see  Repositories#PROTOCOL_SETTING
          */
-        Setting<Protocol> PROTOCOL_SETTING = new Setting<>("protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), false, Setting.Scope.CLUSTER);
+        Setting<Protocol> PROTOCOL_SETTING =
+            new Setting<>("protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope);
         /**
          * region
          * @see  Repositories#REGION_SETTING
          */
-        Setting<String> REGION_SETTING = new Setting<>("region", "", s -> s.toLowerCase(Locale.ROOT), false, Setting.Scope.CLUSTER);
+        Setting<String> REGION_SETTING = new Setting<>("region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
         /**
          * server_side_encryption
          * @see  Repositories#SERVER_SIDE_ENCRYPTION_SETTING
          */
-        Setting<Boolean> SERVER_SIDE_ENCRYPTION_SETTING = Setting.boolSetting("server_side_encryption", false, false, Setting.Scope.CLUSTER);
+        Setting<Boolean> SERVER_SIDE_ENCRYPTION_SETTING =
+            Setting.boolSetting("server_side_encryption", false, Property.NodeScope);
         /**
          * buffer_size
          * @see  Repositories#BUFFER_SIZE_SETTING
          */
-        Setting<ByteSizeValue> BUFFER_SIZE_SETTING = Setting.byteSizeSetting("buffer_size", S3BlobStore.MIN_BUFFER_SIZE, false, Setting.Scope.CLUSTER);
+        Setting<ByteSizeValue> BUFFER_SIZE_SETTING =
+            Setting.byteSizeSetting("buffer_size", new ByteSizeValue(100, ByteSizeUnit.MB),
+                new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope);
         /**
          * max_retries
          * @see  Repositories#MAX_RETRIES_SETTING
          */
-        Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("max_retries", 3, false, Setting.Scope.CLUSTER);
+        Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("max_retries", 3, Property.NodeScope);
         /**
          * chunk_size
          * @see  Repositories#CHUNK_SIZE_SETTING
          */
-        Setting<ByteSizeValue> CHUNK_SIZE_SETTING = Setting.byteSizeSetting("chunk_size", "-1", false, Setting.Scope.CLUSTER);
+        Setting<ByteSizeValue> CHUNK_SIZE_SETTING =
+            Setting.byteSizeSetting("chunk_size", new ByteSizeValue(1, ByteSizeUnit.GB),
+                new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope);
         /**
          * compress
          * @see  Repositories#COMPRESS_SETTING
          */
-        Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("compress", false, false, Setting.Scope.CLUSTER);
+        Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("compress", false, Property.NodeScope);
         /**
          * storage_class
          * @see  Repositories#STORAGE_CLASS_SETTING
          */
-        Setting<String> STORAGE_CLASS_SETTING = Setting.simpleString("storage_class", false, Setting.Scope.CLUSTER);
+        Setting<String> STORAGE_CLASS_SETTING = Setting.simpleString("storage_class", Property.NodeScope);
         /**
          * canned_acl
          * @see  Repositories#CANNED_ACL_SETTING
          */
-        Setting<String> CANNED_ACL_SETTING = Setting.simpleString("canned_acl", false, Setting.Scope.CLUSTER);
+        Setting<String> CANNED_ACL_SETTING = Setting.simpleString("canned_acl", Property.NodeScope);
         /**
          * base_path
          * @see  Repositories#BASE_PATH_SETTING
          */
-        Setting<String> BASE_PATH_SETTING = Setting.simpleString("base_path", false, Setting.Scope.CLUSTER);
+        Setting<String> BASE_PATH_SETTING = Setting.simpleString("base_path", Property.NodeScope);
     }
 
     private final S3BlobStore blobStore;
@@ -248,6 +265,12 @@ public class S3Repository extends BlobStoreRepository {
         this.chunkSize = getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING);
         this.compress = getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING);
 
+        // We make sure that chunkSize is bigger or equal than/to bufferSize
+        if (this.chunkSize.getBytes() < bufferSize.getBytes()) {
+            throw new RepositoryException(name.name(), Repository.CHUNK_SIZE_SETTING.getKey() + " (" + this.chunkSize +
+                ") can't be lower than " + Repository.BUFFER_SIZE_SETTING.getKey() + " (" + bufferSize + ").");
+        }
+
         // Parse and validate the user's S3 Storage Class setting
         String storageClass = getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING);
         String cannedACL = getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING);
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java
index 7d881e0dd30..c5c617e8591 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java
@@ -21,12 +21,20 @@ package org.elasticsearch.cloud.aws;
 
 import com.amazonaws.Protocol;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.repositories.RepositoryException;
+import org.elasticsearch.repositories.RepositoryName;
 import org.elasticsearch.repositories.RepositorySettings;
+import org.elasticsearch.repositories.s3.S3Repository;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
+
 import static org.elasticsearch.repositories.s3.S3Repository.Repositories;
 import static org.elasticsearch.repositories.s3.S3Repository.Repository;
 import static org.elasticsearch.repositories.s3.S3Repository.getValue;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.isEmptyString;
 
@@ -110,9 +118,9 @@ public class RepositoryS3SettingsTests extends ESTestCase {
         assertThat(AwsS3Service.CLOUD_S3.SIGNER_SETTING.get(nodeSettings), is("global-signer"));
         assertThat(getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING),
             is(false));
-        assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(5L));
+        assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(100L));
         assertThat(getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING), is(3));
-        assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getMb(), is(100L));
+        assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getGb(), is(1L));
         assertThat(getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING), is(false));
         assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString());
         assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString());
@@ -138,9 +146,9 @@ public class RepositoryS3SettingsTests extends ESTestCase {
         assertThat(AwsS3Service.CLOUD_S3.SIGNER_SETTING.get(nodeSettings), is("s3-signer"));
         assertThat(getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING),
             is(false));
-        assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(5L));
+        assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(100L));
         assertThat(getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING), is(3));
-        assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getMb(), is(100L));
+        assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getGb(), is(1L));
         assertThat(getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING), is(false));
         assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString());
         assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString());
@@ -292,6 +300,25 @@ public class RepositoryS3SettingsTests extends ESTestCase {
         assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), is("repository-basepath"));
     }
 
+    /**
+     * We test wrong Chunk and Buffer settings
+     */
+    public void testInvalidChunkBufferSizeRepositorySettings() throws IOException {
+        // chunk < buffer should fail
+        internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(10, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.MB),
+            "chunk_size (5mb) can't be lower than buffer_size (10mb).");
+        // chunk > buffer should pass
+        internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(10, ByteSizeUnit.MB), null);
+        // chunk = buffer should pass
+        internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.MB), null);
+        // buffer < 5mb should fail
+        internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(4, ByteSizeUnit.MB), new ByteSizeValue(10, ByteSizeUnit.MB),
+            "Failed to parse value [4mb] for setting [buffer_size] must be >= 5mb");
+        // chunk > 5tb should fail
+        internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(6, ByteSizeUnit.TB),
+            "Failed to parse value [6tb] for setting [chunk_size] must be =< 5tb");
+    }
+
     private Settings buildSettings(Settings... global) {
         Settings.Builder builder = Settings.builder();
         for (Settings settings : global) {
@@ -299,4 +326,28 @@ public class RepositoryS3SettingsTests extends ESTestCase {
         }
         return builder.build();
     }
+
+    private void internalTestInvalidChunkBufferSizeSettings(ByteSizeValue buffer, ByteSizeValue chunk, String expectedMessage)
+        throws IOException {
+        Settings nodeSettings = buildSettings(AWS, S3, REPOSITORIES);
+        RepositorySettings s3RepositorySettings =  new RepositorySettings(nodeSettings, Settings.builder()
+            .put(Repository.BUFFER_SIZE_SETTING.getKey(), buffer)
+            .put(Repository.CHUNK_SIZE_SETTING.getKey(), chunk)
+            .build());
+
+        try {
+            new S3Repository(new RepositoryName("s3", "s3repo"), s3RepositorySettings, null, null);
+            fail("We should either raise a NPE or a RepositoryException or a IllegalArgumentException");
+        } catch (RepositoryException e) {
+            assertThat(e.getDetailedMessage(), containsString(expectedMessage));
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), containsString(expectedMessage));
+        } catch (NullPointerException e) {
+            // Because we passed to the CTOR a Null AwsS3Service, we get a NPE which is expected
+            // in the context of this test
+            if (expectedMessage != null) {
+                fail("We should have raised a RepositoryException");
+            }
+        }
+    }
 }
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java
index f023b64211f..0518d657e23 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.cloud.aws.blobstore;
 
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.ByteArrayOutputStream;
@@ -33,7 +35,7 @@ import static org.hamcrest.Matchers.is;
  * Unit test for {@link S3OutputStream}.
  */
 public class S3OutputStreamTests extends ESTestCase {
-    private static final int BUFFER_SIZE = S3BlobStore.MIN_BUFFER_SIZE.bytesAsInt();
+    private static final int BUFFER_SIZE = new ByteSizeValue(5, ByteSizeUnit.MB).bytesAsInt();
 
     public void testWriteLessDataThanBufferSize() throws IOException {
         MockDefaultS3OutputStream out = newS3OutputStream(BUFFER_SIZE);
diff --git a/qa/backwards-5.0/build.gradle b/qa/backwards-5.0/build.gradle
new file mode 100644
index 00000000000..c86b21dfea4
--- /dev/null
+++ b/qa/backwards-5.0/build.gradle
@@ -0,0 +1,23 @@
+apply plugin: 'elasticsearch.rest-test'
+
+/* This project runs the core REST tests against a 2 node cluster where one of the nodes has a different minor.
+ * Since we don't have a version to test against we currently use the hardcoded snapshot for to bascially run
+ * against ourself. To test that useing a different version works got into distribution/zip and execute:
+ *   gradle clean install -Dbuild.snapshot=false
+ *
+ * This installs the release-build into a local .m2 repository, then change this version here to:
+ *   bwcVersion = "5.0.0"
+ *
+ * now you can run the bwc tests with:
+ *   gradle check -Drepos.mavenlocal=true
+ *
+ * (-Drepos.mavenlocal=true will force gradle to look for the zip distribuiton in the local .m2 repository)
+ */
+integTest {
+  includePackaged = true
+  cluster {
+    numNodes = 2
+    numBwcNodes = 1
+    bwcVersion = "5.0.0-alpha1-SNAPSHOT" // this is the same as the current version until we released the first RC
+  }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java b/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/MultiNodeBackwardsIT.java
similarity index 50%
rename from core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java
rename to qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/MultiNodeBackwardsIT.java
index a7aa3fd60b6..1f3ad15d1bf 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java
+++ b/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/MultiNodeBackwardsIT.java
@@ -16,15 +16,28 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.search.suggest;
 
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.IndexFieldDataService;
-import org.elasticsearch.index.mapper.MapperService;
+package org.elasticsearch.backwards;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+import org.apache.lucene.util.TimeUnits;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.RestTestCandidate;
+import org.elasticsearch.test.rest.parser.RestTestParseException;
 
 import java.io.IOException;
 
-public interface SuggestContextParser {
-    SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService indexFieldDataService) throws IOException;
+@TimeoutSuite(millis = 40 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell
+public class MultiNodeBackwardsIT extends ESRestTestCase {
 
+    public MultiNodeBackwardsIT(RestTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException, RestTestParseException {
+        return createParameters(0, 1);
+    }
 }
+
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/BootstrapCliParserTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/BootstrapCliParserTests.java
deleted file mode 100644
index c172999e7cb..00000000000
--- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/BootstrapCliParserTests.java
+++ /dev/null
@@ -1,273 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.bootstrap;
-
-import org.elasticsearch.Build;
-import org.elasticsearch.Version;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.cli.CliTool.ExitStatus;
-import org.elasticsearch.common.cli.CliToolTestCase;
-import org.elasticsearch.common.cli.MockTerminal;
-import org.elasticsearch.common.cli.UserError;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.monitor.jvm.JvmInfo;
-import org.junit.After;
-import org.junit.Before;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-import static org.elasticsearch.common.cli.CliTool.ExitStatus.OK;
-import static org.elasticsearch.common.cli.CliTool.ExitStatus.OK_AND_EXIT;
-import static org.elasticsearch.common.cli.CliTool.ExitStatus.USAGE;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.nullValue;
-
-@SuppressForbidden(reason = "modifies system properties intentionally")
-public class BootstrapCliParserTests extends CliToolTestCase {
-
-    private MockTerminal terminal = new MockTerminal();
-    private List<String> propertiesToClear = new ArrayList<>();
-    private Map<Object, Object> properties;
-
-    @Before
-    public void before() {
-        this.properties = new HashMap<>(System.getProperties());
-    }
-
-    @After
-    public void clearProperties() {
-        for (String property : propertiesToClear) {
-            System.clearProperty(property);
-        }
-        propertiesToClear.clear();
-        assertEquals("properties leaked", properties, new HashMap<>(System.getProperties()));
-    }
-
-    public void testThatVersionIsReturned() throws Exception {
-        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
-        ExitStatus status = parser.execute(args("version"));
-        assertStatus(status, OK_AND_EXIT);
-
-        String output = terminal.getOutput();
-        assertTrue(output, output.contains(Version.CURRENT.toString()));
-        assertTrue(output, output.contains(Build.CURRENT.shortHash()));
-        assertTrue(output, output.contains(Build.CURRENT.date()));
-        assertTrue(output, output.contains(JvmInfo.jvmInfo().version()));
-    }
-
-    public void testThatVersionIsReturnedAsStartParameter() throws Exception {
-        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
-        ExitStatus status = parser.execute(args("start -V"));
-        assertStatus(status, OK_AND_EXIT);
-
-        String output = terminal.getOutput();
-        assertTrue(output, output.contains(Version.CURRENT.toString()));
-        assertTrue(output, output.contains(Build.CURRENT.shortHash()));
-        assertTrue(output, output.contains(Build.CURRENT.date()));
-        assertTrue(output, output.contains(JvmInfo.jvmInfo().version()));
-
-        terminal.resetOutput();
-        parser = new BootstrapCLIParser(terminal);
-        status = parser.execute(args("start --version"));
-        assertStatus(status, OK_AND_EXIT);
-
-        output = terminal.getOutput();
-        assertTrue(output, output.contains(Version.CURRENT.toString()));
-        assertTrue(output, output.contains(Build.CURRENT.shortHash()));
-        assertTrue(output, output.contains(Build.CURRENT.date()));
-        assertTrue(output, output.contains(JvmInfo.jvmInfo().version()));
-    }
-
-    public void testThatPidFileCanBeConfigured() throws Exception {
-        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
-        registerProperties("es.pidfile");
-
-        ExitStatus status = parser.execute(args("start --pidfile")); // missing pid file
-        assertStatus(status, USAGE);
-
-        // good cases
-        status = parser.execute(args("start --pidfile /tmp/pid"));
-        assertStatus(status, OK);
-        assertSystemProperty("es.pidfile", "/tmp/pid");
-
-        System.clearProperty("es.pidfile");
-        status = parser.execute(args("start -p /tmp/pid"));
-        assertStatus(status, OK);
-        assertSystemProperty("es.pidfile", "/tmp/pid");
-    }
-
-    public void testThatParsingDaemonizeWorks() throws Exception {
-        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
-        registerProperties("es.foreground");
-
-        ExitStatus status = parser.execute(args("start -d"));
-        assertStatus(status, OK);
-        assertThat(System.getProperty("es.foreground"), is("false"));
-    }
-
-    public void testThatNotDaemonizingDoesNotConfigureProperties() throws Exception {
-        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
-        registerProperties("es.foreground");
-
-        ExitStatus status = parser.execute(args("start"));
-        assertStatus(status, OK);
-        assertThat(System.getProperty("es.foreground"), is(nullValue()));
-    }
-
-    public void testThatJavaPropertyStyleArgumentsCanBeParsed() throws Exception {
-        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
-        registerProperties("es.foo", "es.spam");
-
-        ExitStatus status = parser.execute(args("start -Dfoo=bar -Dspam=eggs"));
-        assertStatus(status, OK);
-        assertSystemProperty("es.foo", "bar");
-        assertSystemProperty("es.spam", "eggs");
-    }
-
-    public void testThatJavaPropertyStyleArgumentsWithEsPrefixAreNotPrefixedTwice() throws Exception {
-        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
-        registerProperties("es.spam", "es.pidfile");
-
-        ExitStatus status = parser.execute(args("start -Des.pidfile=/path/to/foo/elasticsearch/distribution/zip/target/integ-tests/es.pid -Dspam=eggs"));
-        assertStatus(status, OK);
-        assertThat(System.getProperty("es.es.pidfile"), is(nullValue()));
-        assertSystemProperty("es.pidfile", "/path/to/foo/elasticsearch/distribution/zip/target/integ-tests/es.pid");
-        assertSystemProperty("es.spam", "eggs");
-    }
-
-    public void testThatUnknownLongOptionsCanBeParsed() throws Exception {
-        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
-        registerProperties("es.network.host", "es.my.option");
-
-        ExitStatus status = parser.execute(args("start --network.host 127.0.0.1 --my.option=true"));
-        assertStatus(status, OK);
-        assertSystemProperty("es.network.host", "127.0.0.1");
-        assertSystemProperty("es.my.option", "true");
-    }
-
-    public void testThatUnknownLongOptionsNeedAValue() throws Exception {
-        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
-        registerProperties("es.network.host");
-
-        ExitStatus status = parser.execute(args("start --network.host"));
-        assertStatus(status, USAGE);
-        String output = terminal.getOutput();
-        assertTrue(output, output.contains("Parameter [network.host] needs value"));
-
-        terminal.resetOutput();
-        status = parser.execute(args("start --network.host --foo"));
-        assertStatus(status, USAGE);
-        output = terminal.getOutput();
-        assertTrue(output, output.contains("Parameter [network.host] needs value"));
-    }
-
-    public void testParsingErrors() throws Exception {
-        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
-
-        // unknown params
-        ExitStatus status = parser.execute(args("version --unknown-param /tmp/pid"));
-        assertStatus(status, USAGE);
-        String output = terminal.getOutput();
-        assertTrue(output, output.contains("Unrecognized option: --unknown-param"));
-
-        // single dash in extra params
-        terminal.resetOutput();
-        parser = new BootstrapCLIParser(terminal);
-        status = parser.execute(args("start -network.host 127.0.0.1"));
-        assertStatus(status, USAGE);
-        output = terminal.getOutput();
-        assertTrue(output, output.contains("Parameter [-network.host]does not start with --"));
-
-        // never ended parameter
-        terminal = new MockTerminal();
-        parser = new BootstrapCLIParser(terminal);
-        status = parser.execute(args("start --network.host"));
-        assertStatus(status, USAGE);
-        output = terminal.getOutput();
-        assertTrue(output, output.contains("Parameter [network.host] needs value"));
-
-        // free floating value
-        terminal = new MockTerminal();
-        parser = new BootstrapCLIParser(terminal);
-        status = parser.execute(args("start 127.0.0.1"));
-        assertStatus(status, USAGE);
-        output = terminal.getOutput();
-        assertTrue(output, output.contains("Parameter [127.0.0.1]does not start with --"));
-    }
-
-    public void testHelpWorks() throws Exception {
-        List<Tuple<String, String>> tuples = new ArrayList<>();
-        tuples.add(new Tuple<>("version --help", "elasticsearch-version.help"));
-        tuples.add(new Tuple<>("version -h", "elasticsearch-version.help"));
-        tuples.add(new Tuple<>("start --help", "elasticsearch-start.help"));
-        tuples.add(new Tuple<>("start -h", "elasticsearch-start.help"));
-        tuples.add(new Tuple<>("--help", "elasticsearch.help"));
-        tuples.add(new Tuple<>("-h", "elasticsearch.help"));
-
-        for (Tuple<String, String> tuple : tuples) {
-            terminal.resetOutput();
-            BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
-            ExitStatus status = parser.execute(args(tuple.v1()));
-            assertStatus(status, OK_AND_EXIT);
-            assertTerminalOutputContainsHelpFile(terminal, "/org/elasticsearch/bootstrap/" + tuple.v2());
-        }
-    }
-
-    public void testThatSpacesInParametersAreSupported() throws Exception {
-        // emulates: bin/elasticsearch --node.name "'my node with spaces'" --pidfile "'/tmp/my pid.pid'"
-        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
-        registerProperties("es.pidfile", "es.my.param");
-
-        ExitStatus status = parser.execute("start", "--pidfile", "foo with space", "--my.param", "my awesome neighbour");
-        assertStatus(status, OK);
-        assertSystemProperty("es.pidfile", "foo with space");
-        assertSystemProperty("es.my.param", "my awesome neighbour");
-
-    }
-
-    public void testThatHelpfulErrorMessageIsGivenWhenParametersAreOutOfOrder() throws Exception {
-        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
-        UserError e = expectThrows(UserError.class, () -> {
-                parser.parse("start", new String[]{"--foo=bar", "-Dbaz=qux"});
-        });
-        assertThat(e.getMessage(), containsString("must be before any parameters starting with --"));
-        assertNull(System.getProperty("es.foo"));
-    }
-
-    private void registerProperties(String ... systemProperties) {
-        propertiesToClear.addAll(Arrays.asList(systemProperties));
-    }
-
-    private void assertSystemProperty(String name, String expectedValue) throws Exception {
-        String msg = String.format(Locale.ROOT, "Expected property %s to be %s, terminal output was %s", name, expectedValue, terminal.getOutput());
-        assertThat(msg, System.getProperty(name), is(expectedValue));
-    }
-
-    private void assertStatus(ExitStatus status, ExitStatus expectedStatus) throws Exception {
-        assertThat(String.format(Locale.ROOT, "Expected status to be [%s], but was [%s], terminal output was %s", expectedStatus, status, terminal.getOutput()), status, is(expectedStatus));
-    }
-}
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJNANativesTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJNANativesTests.java
index 080eee2501b..1ef5bdfcfae 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJNANativesTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJNANativesTests.java
@@ -27,7 +27,9 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.util.List;
 
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 public class EvilJNANativesTests extends ESTestCase {
 
@@ -49,4 +51,26 @@ public class EvilJNANativesTests extends ESTestCase {
             assertThat(JNANatives.MAX_NUMBER_OF_THREADS, equalTo(-1L));
         }
     }
+
+    public void testSetMaxSizeVirtualMemory() throws IOException {
+        if (Constants.LINUX) {
+            final List<String> lines = Files.readAllLines(PathUtils.get("/proc/self/limits"));
+            if (!lines.isEmpty()) {
+                for (String line : lines) {
+                    if (line != null && line.startsWith("Max address space")) {
+                        final String[] fields = line.split("\\s+");
+                        final String limit = fields[3];
+                        assertEquals(JNANatives.rlimitToString(JNANatives.MAX_SIZE_VIRTUAL_MEMORY), limit);
+                        return;
+                    }
+                }
+            }
+            fail("should have read max size virtual memory from /proc/self/limits");
+        } else if (Constants.MAC_OS_X) {
+            assertThat(JNANatives.MAX_SIZE_VIRTUAL_MEMORY, anyOf(equalTo(Long.MIN_VALUE), greaterThanOrEqualTo(0L)));
+        } else {
+            assertThat(JNANatives.MAX_SIZE_VIRTUAL_MEMORY, equalTo(Long.MIN_VALUE));
+        }
+    }
+
 }
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/common/cli/CheckFileCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/cli/CheckFileCommandTests.java
deleted file mode 100644
index 485886b5cf4..00000000000
--- a/qa/evil-tests/src/test/java/org/elasticsearch/common/cli/CheckFileCommandTests.java
+++ /dev/null
@@ -1,329 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.cli;
-
-import com.google.common.jimfs.Configuration;
-import com.google.common.jimfs.Jimfs;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.test.ESTestCase;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.FileSystem;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.attribute.GroupPrincipal;
-import java.nio.file.attribute.PosixFileAttributeView;
-import java.nio.file.attribute.PosixFileAttributes;
-import java.nio.file.attribute.PosixFilePermission;
-import java.nio.file.attribute.UserPrincipal;
-import java.util.Set;
-
-import static org.hamcrest.Matchers.allOf;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.is;
-
-/**
- *
- */
-public class CheckFileCommandTests extends ESTestCase {
-
-    private MockTerminal captureOutputTerminal = new MockTerminal();
-
-    private Configuration jimFsConfiguration = Configuration.unix().toBuilder().setAttributeViews("basic", "owner", "posix", "unix").build();
-    private Configuration jimFsConfigurationWithoutPermissions = randomBoolean() ? Configuration.unix().toBuilder().setAttributeViews("basic").build() : Configuration.windows();
-
-    private enum Mode {
-        CHANGE, KEEP, DISABLED
-    }
-
-    public void testThatCommandLogsErrorMessageOnFail() throws Exception {
-        executeCommand(jimFsConfiguration, new PermissionCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.CHANGE));
-        assertThat(captureOutputTerminal.getOutput(), containsString("Please ensure that the user account running Elasticsearch has read access to this file"));
-    }
-
-    public void testThatCommandLogsNothingWhenPermissionRemains() throws Exception {
-        executeCommand(jimFsConfiguration, new PermissionCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.KEEP));
-        assertTrue(captureOutputTerminal.getOutput().isEmpty());
-    }
-
-    public void testThatCommandLogsNothingWhenDisabled() throws Exception {
-        executeCommand(jimFsConfiguration, new PermissionCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.DISABLED));
-        assertTrue(captureOutputTerminal.getOutput().isEmpty());
-    }
-
-    public void testThatCommandLogsNothingIfFilesystemDoesNotSupportPermissions() throws Exception {
-        executeCommand(jimFsConfigurationWithoutPermissions, new PermissionCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.DISABLED));
-        assertTrue(captureOutputTerminal.getOutput().isEmpty());
-    }
-
-    public void testThatCommandLogsOwnerChange() throws Exception {
-        executeCommand(jimFsConfiguration, new OwnerCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.CHANGE));
-        assertThat(captureOutputTerminal.getOutput(), allOf(containsString("Owner of file ["), containsString("] used to be ["), containsString("], but now is [")));
-    }
-
-    public void testThatCommandLogsNothingIfOwnerRemainsSame() throws Exception {
-        executeCommand(jimFsConfiguration, new OwnerCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.KEEP));
-        assertTrue(captureOutputTerminal.getOutput().isEmpty());
-    }
-
-    public void testThatCommandLogsNothingIfOwnerIsDisabled() throws Exception {
-        executeCommand(jimFsConfiguration, new OwnerCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.DISABLED));
-        assertTrue(captureOutputTerminal.getOutput().isEmpty());
-    }
-
-    public void testThatCommandLogsNothingIfFileSystemDoesNotSupportOwners() throws Exception {
-        executeCommand(jimFsConfigurationWithoutPermissions, new OwnerCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.DISABLED));
-        assertTrue(captureOutputTerminal.getOutput().isEmpty());
-    }
-
-    public void testThatCommandLogsIfGroupChanges() throws Exception {
-        executeCommand(jimFsConfiguration, new GroupCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.CHANGE));
-        assertThat(captureOutputTerminal.getOutput(), allOf(containsString("Group of file ["), containsString("] used to be ["), containsString("], but now is [")));
-    }
-
-    public void testThatCommandLogsNothingIfGroupRemainsSame() throws Exception {
-        executeCommand(jimFsConfiguration, new GroupCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.KEEP));
-        assertTrue(captureOutputTerminal.getOutput().isEmpty());
-    }
-
-    public void testThatCommandLogsNothingIfGroupIsDisabled() throws Exception {
-        executeCommand(jimFsConfiguration, new GroupCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.DISABLED));
-        assertTrue(captureOutputTerminal.getOutput().isEmpty());
-    }
-
-    public void testThatCommandLogsNothingIfFileSystemDoesNotSupportGroups() throws Exception {
-        executeCommand(jimFsConfigurationWithoutPermissions, new GroupCheckFileCommand(createTempDir(), captureOutputTerminal, Mode.DISABLED));
-        assertTrue(captureOutputTerminal.getOutput().isEmpty());
-    }
-
-    public void testThatCommandDoesNotLogAnythingOnFileCreation() throws Exception {
-        Configuration configuration = randomBoolean() ? jimFsConfiguration : jimFsConfigurationWithoutPermissions;
-
-        try (FileSystem fs = Jimfs.newFileSystem(configuration)) {
-            Path path = fs.getPath(randomAsciiOfLength(10));
-            Settings settings = Settings.builder()
-                    .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                    .build();
-            new CreateFileCommand(captureOutputTerminal, path).execute(settings, new Environment(settings));
-            assertThat(Files.exists(path), is(true));
-        }
-
-        assertTrue(captureOutputTerminal.getOutput().isEmpty());
-    }
-
-    public void testThatCommandWorksIfFileIsDeletedByCommand() throws Exception {
-        Configuration configuration = randomBoolean() ? jimFsConfiguration : jimFsConfigurationWithoutPermissions;
-
-        try (FileSystem fs = Jimfs.newFileSystem(configuration)) {
-            Path path = fs.getPath(randomAsciiOfLength(10));
-            Files.write(path, "anything".getBytes(StandardCharsets.UTF_8));
-
-            Settings settings = Settings.builder()
-                    .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                    .build();
-            new DeleteFileCommand(captureOutputTerminal, path).execute(settings, new Environment(settings));
-            assertThat(Files.exists(path), is(false));
-        }
-
-        assertTrue(captureOutputTerminal.getOutput().isEmpty());
-    }
-
-    private void executeCommand(Configuration configuration, AbstractTestCheckFileCommand command) throws Exception {
-        try (FileSystem fs = Jimfs.newFileSystem(configuration)) {
-            command.execute(fs);
-        }
-    }
-
-    abstract class AbstractTestCheckFileCommand extends CheckFileCommand {
-
-        protected final Mode mode;
-        protected FileSystem fs;
-        protected Path[] paths;
-        final Path baseDir;
-
-        public AbstractTestCheckFileCommand(Path baseDir, Terminal terminal, Mode mode) throws IOException {
-            super(terminal);
-            this.mode = mode;
-            this.baseDir = baseDir;
-        }
-
-        public CliTool.ExitStatus execute(FileSystem fs) throws Exception {
-            this.fs = fs;
-            this.paths = new Path[] { writePath(fs, "p1", "anything"), writePath(fs, "p2", "anything"), writePath(fs, "p3", "anything") };
-            Settings settings = Settings.settingsBuilder()
-                    .put(Environment.PATH_HOME_SETTING.getKey(), baseDir.toString())
-                    .build();
-            return super.execute(Settings.EMPTY, new Environment(settings));
-        }
-
-        private Path writePath(FileSystem fs, String name, String content) throws IOException {
-            Path path = fs.getPath(name);
-            Files.write(path, content.getBytes(StandardCharsets.UTF_8));
-            return path;
-        }
-
-        @Override
-        protected Path[] pathsForPermissionsCheck(Settings settings, Environment env) {
-            return paths;
-        }
-    }
-
-    /**
-     * command that changes permissions from a file if enabled
-     */
-    class PermissionCheckFileCommand extends AbstractTestCheckFileCommand {
-
-        public PermissionCheckFileCommand(Path baseDir, Terminal terminal, Mode mode) throws IOException {
-            super(baseDir, terminal, mode);
-        }
-
-        @Override
-        public CliTool.ExitStatus doExecute(Settings settings, Environment env) throws Exception {
-            int randomInt = randomInt(paths.length - 1);
-            Path randomPath = paths[randomInt];
-            switch (mode) {
-                case CHANGE:
-                    Files.write(randomPath, randomAsciiOfLength(10).getBytes(StandardCharsets.UTF_8));
-                    Files.setPosixFilePermissions(randomPath, Sets.newHashSet(PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.OTHERS_EXECUTE, PosixFilePermission.GROUP_EXECUTE));
-                    break;
-                case KEEP:
-                    Files.write(randomPath, randomAsciiOfLength(10).getBytes(StandardCharsets.UTF_8));
-                    Set<PosixFilePermission> posixFilePermissions = Files.getPosixFilePermissions(randomPath);
-                    Files.setPosixFilePermissions(randomPath, posixFilePermissions);
-                    break;
-            }
-            return CliTool.ExitStatus.OK;
-        }
-
-    }
-
-    /**
-     * command that changes the owner of a file if enabled
-     */
-    class OwnerCheckFileCommand extends AbstractTestCheckFileCommand {
-
-        public OwnerCheckFileCommand(Path baseDir, Terminal terminal, Mode mode) throws IOException {
-            super(baseDir, terminal, mode);
-        }
-
-        @Override
-        public CliTool.ExitStatus doExecute(Settings settings, Environment env) throws Exception {
-            int randomInt = randomInt(paths.length - 1);
-            Path randomPath = paths[randomInt];
-            switch (mode) {
-                case CHANGE:
-                    Files.write(randomPath, randomAsciiOfLength(10).getBytes(StandardCharsets.UTF_8));
-                    UserPrincipal randomOwner = fs.getUserPrincipalLookupService().lookupPrincipalByName(randomAsciiOfLength(10));
-                    Files.setOwner(randomPath, randomOwner);
-                    break;
-                case KEEP:
-                    Files.write(randomPath, randomAsciiOfLength(10).getBytes(StandardCharsets.UTF_8));
-                    UserPrincipal originalOwner = Files.getOwner(randomPath);
-                    Files.setOwner(randomPath, originalOwner);
-                    break;
-            }
-
-            return CliTool.ExitStatus.OK;
-        }
-    }
-
-    /**
-     * command that changes the group of a file if enabled
-     */
-    class GroupCheckFileCommand extends AbstractTestCheckFileCommand {
-
-        public GroupCheckFileCommand(Path baseDir, Terminal terminal, Mode mode) throws IOException {
-            super(baseDir, terminal, mode);
-        }
-
-        @Override
-        public CliTool.ExitStatus doExecute(Settings settings, Environment env) throws Exception {
-            int randomInt = randomInt(paths.length - 1);
-            Path randomPath = paths[randomInt];
-            switch (mode) {
-                case CHANGE:
-                    Files.write(randomPath, randomAsciiOfLength(10).getBytes(StandardCharsets.UTF_8));
-                    GroupPrincipal randomPrincipal = fs.getUserPrincipalLookupService().lookupPrincipalByGroupName(randomAsciiOfLength(10));
-                    Files.getFileAttributeView(randomPath, PosixFileAttributeView.class).setGroup(randomPrincipal);
-                    break;
-                case KEEP:
-                    Files.write(randomPath, randomAsciiOfLength(10).getBytes(StandardCharsets.UTF_8));
-                    GroupPrincipal groupPrincipal = Files.readAttributes(randomPath, PosixFileAttributes.class).group();
-                    Files.getFileAttributeView(randomPath, PosixFileAttributeView.class).setGroup(groupPrincipal);
-                    break;
-            }
-
-            return CliTool.ExitStatus.OK;
-        }
-    }
-
-    /**
-     * A command that creates a non existing file
-     */
-    class CreateFileCommand extends CheckFileCommand {
-
-        private final Path pathToCreate;
-
-        public CreateFileCommand(Terminal terminal, Path pathToCreate) {
-            super(terminal);
-            this.pathToCreate = pathToCreate;
-        }
-
-        @Override
-        public CliTool.ExitStatus doExecute(Settings settings, Environment env) throws Exception {
-            Files.write(pathToCreate, "anything".getBytes(StandardCharsets.UTF_8));
-            return CliTool.ExitStatus.OK;
-        }
-
-        @Override
-        protected Path[] pathsForPermissionsCheck(Settings settings, Environment env) throws Exception {
-            return new Path[] { pathToCreate };
-        }
-    }
-
-    /**
-     * A command that deletes an existing file
-     */
-    class DeleteFileCommand extends CheckFileCommand {
-
-        private final Path pathToDelete;
-
-        public DeleteFileCommand(Terminal terminal, Path pathToDelete) {
-            super(terminal);
-            this.pathToDelete = pathToDelete;
-        }
-
-        @Override
-        public CliTool.ExitStatus doExecute(Settings settings, Environment env) throws Exception {
-            Files.delete(pathToDelete);
-            return CliTool.ExitStatus.OK;
-        }
-
-        @Override
-        protected Path[] pathsForPermissionsCheck(Settings settings, Environment env) throws Exception {
-            return new Path[] {pathToDelete};
-        }
-    }
-}
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/common/cli/CliToolTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/cli/CliToolTests.java
deleted file mode 100644
index 144a12f141b..00000000000
--- a/qa/evil-tests/src/test/java/org/elasticsearch/common/cli/CliToolTests.java
+++ /dev/null
@@ -1,349 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.cli;
-
-import org.apache.commons.cli.CommandLine;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-
-import static java.util.Collections.unmodifiableMap;
-import static org.elasticsearch.common.cli.CliTool.ExitStatus.OK;
-import static org.elasticsearch.common.cli.CliTool.ExitStatus.USAGE;
-import static org.elasticsearch.common.cli.CliToolConfig.Builder.cmd;
-import static org.hamcrest.Matchers.arrayContaining;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.is;
-
-@SuppressForbidden(reason = "modifies system properties intentionally")
-public class CliToolTests extends CliToolTestCase {
-    public void testOK() throws Exception {
-        Terminal terminal = new MockTerminal();
-        final AtomicReference<Boolean> executed = new AtomicReference<>(false);
-        final NamedCommand cmd = new NamedCommand("cmd", terminal) {
-            @Override
-            public CliTool.ExitStatus execute(Settings settings, Environment env) {
-                executed.set(true);
-                return OK;
-            }
-        };
-        SingleCmdTool tool = new SingleCmdTool("tool", terminal, cmd);
-        CliTool.ExitStatus status = tool.execute();
-        assertStatus(status, OK);
-        assertCommandHasBeenExecuted(executed);
-    }
-
-    public void testUsageError() throws Exception {
-        Terminal terminal = new MockTerminal();
-        final AtomicReference<Boolean> executed = new AtomicReference<>(false);
-        final NamedCommand cmd = new NamedCommand("cmd", terminal) {
-            @Override
-            public CliTool.ExitStatus execute(Settings settings, Environment env) throws UserError {
-                executed.set(true);
-                throw new UserError(CliTool.ExitStatus.USAGE, "bad usage");
-            }
-        };
-        SingleCmdTool tool = new SingleCmdTool("tool", terminal, cmd);
-        CliTool.ExitStatus status = tool.execute();
-        assertStatus(status, CliTool.ExitStatus.USAGE);
-        assertCommandHasBeenExecuted(executed);
-    }
-
-    public void testMultiCommand() throws Exception {
-        Terminal terminal = new MockTerminal();
-        int count = randomIntBetween(2, 7);
-        List<AtomicReference<Boolean>> executed = new ArrayList<>(count);
-        for (int i = 0; i < count; i++) {
-            executed.add(new AtomicReference<>(false));
-        }
-        NamedCommand[] cmds = new NamedCommand[count];
-        for (int i = 0; i < count; i++) {
-            final int index = i;
-            cmds[i] = new NamedCommand("cmd" + index, terminal) {
-                @Override
-                public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
-                    executed.get(index).set(true);
-                    return OK;
-                }
-            };
-        }
-        MultiCmdTool tool = new MultiCmdTool("tool", terminal, cmds);
-        int cmdIndex = randomIntBetween(0, count-1);
-        CliTool.ExitStatus status = tool.execute("cmd" + cmdIndex);
-        assertThat(status, is(OK));
-        for (int i = 0; i < count; i++) {
-            assertThat(executed.get(i).get(), is(i == cmdIndex));
-        }
-    }
-
-    public void testMultiCommandUnknownCommand() throws Exception {
-        Terminal terminal = new MockTerminal();
-        int count = randomIntBetween(2, 7);
-        List<AtomicReference<Boolean>> executed = new ArrayList<>(count);
-        for (int i = 0; i < count; i++) {
-            executed.add(new AtomicReference<>(false));
-        }
-        NamedCommand[] cmds = new NamedCommand[count];
-        for (int i = 0; i < count; i++) {
-            final int index = i;
-            cmds[i] = new NamedCommand("cmd" + index, terminal) {
-                @Override
-                public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
-                    executed.get(index).set(true);
-                    return OK;
-                }
-            };
-        }
-        MultiCmdTool tool = new MultiCmdTool("tool", terminal, cmds);
-        CliTool.ExitStatus status = tool.execute("cmd" + count); // "cmd" + count doesn't exist
-        assertThat(status, is(CliTool.ExitStatus.USAGE));
-        for (int i = 0; i < count; i++) {
-            assertThat(executed.get(i).get(), is(false));
-        }
-    }
-
-    public void testSingleCommandToolHelp() throws Exception {
-        MockTerminal terminal = new MockTerminal();
-        final AtomicReference<Boolean> executed = new AtomicReference<>(false);
-        final NamedCommand cmd = new NamedCommand("cmd1", terminal) {
-            @Override
-            public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
-                executed.set(true);
-                throw new IOException("io error");
-            }
-        };
-        SingleCmdTool tool = new SingleCmdTool("tool", terminal, cmd);
-        CliTool.ExitStatus status = tool.execute(args("-h"));
-        assertStatus(status, CliTool.ExitStatus.OK_AND_EXIT);
-        assertThat(terminal.getOutput(), containsString("cmd1 help"));
-    }
-
-    public void testMultiCommandToolHelp() throws Exception {
-        MockTerminal terminal = new MockTerminal();
-        NamedCommand[] cmds = new NamedCommand[2];
-        cmds[0] = new NamedCommand("cmd0", terminal) {
-            @Override
-            public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
-                return OK;
-            }
-        };
-        cmds[1] = new NamedCommand("cmd1", terminal) {
-            @Override
-            public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
-                return OK;
-            }
-        };
-        MultiCmdTool tool = new MultiCmdTool("tool", terminal, cmds);
-        CliTool.ExitStatus status = tool.execute(args("-h"));
-        assertStatus(status, CliTool.ExitStatus.OK_AND_EXIT);
-        assertThat(terminal.getOutput(), containsString("tool help"));
-    }
-
-    public void testMultiCommandCmdHelp() throws Exception {
-        MockTerminal terminal = new MockTerminal();
-        NamedCommand[] cmds = new NamedCommand[2];
-        cmds[0] = new NamedCommand("cmd0", terminal) {
-            @Override
-            public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
-                return OK;
-            }
-        };
-        cmds[1] = new NamedCommand("cmd1", terminal) {
-            @Override
-            public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
-                return OK;
-            }
-        };
-        MultiCmdTool tool = new MultiCmdTool("tool", terminal, cmds);
-        CliTool.ExitStatus status = tool.execute(args("cmd1 -h"));
-        assertStatus(status, CliTool.ExitStatus.OK_AND_EXIT);
-        assertThat(terminal.getOutput(), containsString("cmd1 help"));
-    }
-
-    public void testNonUserErrorPropagates() throws Exception {
-        MockTerminal terminal = new MockTerminal();
-        NamedCommand cmd = new NamedCommand("cmd", terminal) {
-            @Override
-            public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
-                throw new IOException("error message");
-            }
-        };
-        SingleCmdTool tool = new SingleCmdTool("tool", terminal, cmd);
-        IOException e = expectThrows(IOException.class, () -> {
-           tool.execute();
-        });
-        assertEquals("error message", e.getMessage());
-    }
-
-    public void testMultipleLaunch() throws Exception {
-        Terminal terminal = new MockTerminal();
-        final AtomicReference<Boolean> executed = new AtomicReference<>(false);
-        final NamedCommand cmd = new NamedCommand("cmd", terminal) {
-            @Override
-            public CliTool.ExitStatus execute(Settings settings, Environment env) {
-                executed.set(true);
-                return OK;
-            }
-        };
-        SingleCmdTool tool = new SingleCmdTool("tool", terminal, cmd);
-        tool.parse("cmd", Strings.splitStringByCommaToArray("--verbose"));
-        tool.parse("cmd", Strings.splitStringByCommaToArray("--silent"));
-        tool.parse("cmd", Strings.splitStringByCommaToArray("--help"));
-    }
-
-    public void testPromptForSetting() throws Exception {
-        final AtomicReference<String> promptedSecretValue = new AtomicReference<>(null);
-        final AtomicReference<String> promptedTextValue = new AtomicReference<>(null);
-        final MockTerminal terminal = new MockTerminal();
-        terminal.addTextInput("replaced");
-        terminal.addSecretInput("changeit");
-        final NamedCommand cmd = new NamedCommand("noop", terminal) {
-            @Override
-            public CliTool.ExitStatus execute(Settings settings, Environment env) {
-                promptedSecretValue.set(settings.get("foo.password"));
-                promptedTextValue.set(settings.get("replace"));
-                return OK;
-            }
-        };
-
-        System.setProperty("es.foo.password", InternalSettingsPreparer.SECRET_PROMPT_VALUE);
-        System.setProperty("es.replace", InternalSettingsPreparer.TEXT_PROMPT_VALUE);
-        try {
-            new SingleCmdTool("tool", terminal, cmd).execute();
-        } finally {
-            System.clearProperty("es.foo.password");
-            System.clearProperty("es.replace");
-        }
-
-        assertThat(promptedSecretValue.get(), is("changeit"));
-        assertThat(promptedTextValue.get(), is("replaced"));
-    }
-
-    public void testStopAtNonOptionParsing() throws Exception {
-        final CliToolConfig.Cmd lenientCommand = cmd("lenient", CliTool.Command.Exit.class).stopAtNonOption(true).build();
-        final CliToolConfig.Cmd strictCommand = cmd("strict", CliTool.Command.Exit.class).stopAtNonOption(false).build();
-        final CliToolConfig config = CliToolConfig.config("elasticsearch", CliTool.class).cmds(lenientCommand, strictCommand).build();
-
-        MockTerminal terminal = new MockTerminal();
-        final CliTool cliTool = new CliTool(config, terminal) {
-            @Override
-            protected Command parse(String cmdName, CommandLine cli) throws Exception {
-                return new NamedCommand(cmdName, terminal) {
-                    @Override
-                    public ExitStatus execute(Settings settings, Environment env) throws Exception {
-                        return OK;
-                    }
-                };
-            }
-        };
-
-        // known parameters, no error
-        assertStatus(cliTool.execute(args("lenient --verbose")), OK);
-        assertStatus(cliTool.execute(args("lenient -v")), OK);
-
-        // unknown parameters, no error
-        assertStatus(cliTool.execute(args("lenient --unknown")), OK);
-        assertStatus(cliTool.execute(args("lenient -u")), OK);
-
-        // unknown parameters, error
-        assertStatus(cliTool.execute(args("strict --unknown")), USAGE);
-        assertThat(terminal.getOutput(), containsString("Unrecognized option: --unknown"));
-
-        terminal.resetOutput();
-        assertStatus(cliTool.execute(args("strict -u")), USAGE);
-        assertThat(terminal.getOutput(), containsString("Unrecognized option: -u"));
-    }
-
-    private void assertStatus(CliTool.ExitStatus status, CliTool.ExitStatus expectedStatus) {
-        assertThat(status, is(expectedStatus));
-    }
-
-    private void assertCommandHasBeenExecuted(AtomicReference<Boolean> executed) {
-        assertThat("Expected command atomic reference counter to be set to true", executed.get(), is(Boolean.TRUE));
-    }
-
-    private static class SingleCmdTool extends CliTool {
-
-        private final Command command;
-
-        private SingleCmdTool(String name, Terminal terminal, NamedCommand command) {
-            super(CliToolConfig.config(name, SingleCmdTool.class)
-                    .cmds(cmd(command.name, command.getClass()))
-                    .build(), terminal);
-            this.command = command;
-        }
-
-        @Override
-        protected Command parse(String cmdName, CommandLine cli) throws Exception {
-            return command;
-        }
-    }
-
-    private static class MultiCmdTool extends CliTool {
-
-        private final Map<String, Command> commands;
-
-        private MultiCmdTool(String name, Terminal terminal, NamedCommand... commands) {
-            super(CliToolConfig.config(name, MultiCmdTool.class)
-                    .cmds(cmds(commands))
-                    .build(), terminal);
-            Map<String, Command> commandByName = new HashMap<>();
-            for (int i = 0; i < commands.length; i++) {
-                commandByName.put(commands[i].name, commands[i]);
-            }
-            this.commands = unmodifiableMap(commandByName);
-        }
-
-        @Override
-        protected Command parse(String cmdName, CommandLine cli) throws Exception {
-            return commands.get(cmdName);
-        }
-
-        private static CliToolConfig.Cmd[] cmds(NamedCommand... commands) {
-            CliToolConfig.Cmd[] cmds = new CliToolConfig.Cmd[commands.length];
-            for (int i = 0; i < commands.length; i++) {
-                cmds[i] = cmd(commands[i].name, commands[i].getClass()).build();
-            }
-            return cmds;
-        }
-    }
-
-    private static abstract class NamedCommand extends CliTool.Command {
-
-        private final String name;
-
-        private NamedCommand(String name, Terminal terminal) {
-            super(terminal);
-            this.name = name;
-        }
-    }
-}
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
index 86f02fe6f30..106385578e9 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
@@ -19,6 +19,22 @@
 
 package org.elasticsearch.plugins;
 
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.google.common.jimfs.Configuration;
+import com.google.common.jimfs.Jimfs;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.SuppressForbidden;
+import org.elasticsearch.Version;
+import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.cli.UserError;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.io.PathUtilsForTesting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.PosixPermissionsResetter;
+import org.junit.After;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.MalformedURLException;
@@ -26,6 +42,7 @@ import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.DirectoryStream;
 import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.FileSystem;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
@@ -36,49 +53,100 @@ import java.nio.file.attribute.BasicFileAttributes;
 import java.nio.file.attribute.PosixFileAttributeView;
 import java.nio.file.attribute.PosixFileAttributes;
 import java.nio.file.attribute.PosixFilePermission;
+import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
-import org.apache.lucene.util.LuceneTestCase;
-import org.elasticsearch.Version;
-import org.elasticsearch.common.cli.CliTool;
-import org.elasticsearch.common.cli.CliToolTestCase;
-import org.elasticsearch.common.cli.MockTerminal;
-import org.elasticsearch.common.cli.Terminal;
-import org.elasticsearch.common.cli.UserError;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.PosixPermissionsResetter;
-import org.junit.BeforeClass;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 
 @LuceneTestCase.SuppressFileSystems("*")
 public class InstallPluginCommandTests extends ESTestCase {
 
-    private static boolean isPosix;
+    private final Function<String, Path> temp;
 
-    @BeforeClass
-    public static void checkPosix() throws IOException {
-        isPosix = Files.getFileAttributeView(createTempFile(), PosixFileAttributeView.class) != null;
+    private final FileSystem fs;
+    private final boolean isPosix;
+    private final boolean isReal;
+    private final String javaIoTmpdir;
+
+    @SuppressForbidden(reason = "sets java.io.tmpdir")
+    public InstallPluginCommandTests(FileSystem fs, Function<String, Path> temp) {
+        this.fs = fs;
+        this.temp = temp;
+        this.isPosix = fs.supportedFileAttributeViews().contains("posix");
+        this.isReal = fs == PathUtils.getDefaultFileSystem();
+        PathUtilsForTesting.installMock(fs);
+        javaIoTmpdir = System.getProperty("java.io.tmpdir");
+        System.setProperty("java.io.tmpdir", temp.apply("tmpdir").toString());
+    }
+
+    @After
+    @SuppressForbidden(reason = "resets java.io.tmpdir")
+    public void tearDown() throws Exception {
+        System.setProperty("java.io.tmpdir", javaIoTmpdir);
+        PathUtilsForTesting.teardown();
+        super.tearDown();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        class Parameter {
+            private final FileSystem fileSystem;
+            private final Function<String, Path> temp;
+
+            public Parameter(FileSystem fileSystem, String root) {
+                this(fileSystem, s -> {
+                    try {
+                        return Files.createTempDirectory(fileSystem.getPath(root), s);
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                });
+            }
+
+            public Parameter(FileSystem fileSystem, Function<String, Path> temp) {
+                this.fileSystem = fileSystem;
+                this.temp = temp;
+            }
+        }
+        List<Parameter> parameters = new ArrayList<>();
+        parameters.add(new Parameter(Jimfs.newFileSystem(Configuration.windows()), "c:\\"));
+        parameters.add(new Parameter(Jimfs.newFileSystem(toPosix(Configuration.osX())), "/"));
+        parameters.add(new Parameter(Jimfs.newFileSystem(toPosix(Configuration.unix())), "/"));
+        parameters.add(new Parameter(PathUtils.getDefaultFileSystem(), LuceneTestCase::createTempDir ));
+        return parameters.stream().map(p -> new Object[] { p.fileSystem, p.temp }).collect(Collectors.toList());
+    }
+
+    private static Configuration toPosix(Configuration configuration) {
+        return configuration.toBuilder().setAttributeViews("basic", "owner", "posix", "unix").build();
     }
 
     /** Creates a test environment with bin, config and plugins directories. */
-    static Environment createEnv() throws IOException {
-        Path home = createTempDir();
+    static Environment createEnv(FileSystem fs, Function<String, Path> temp) throws IOException {
+        Path home = temp.apply("install-plugin-command-tests");
         Files.createDirectories(home.resolve("bin"));
         Files.createFile(home.resolve("bin").resolve("elasticsearch"));
         Files.createDirectories(home.resolve("config"));
         Files.createFile(home.resolve("config").resolve("elasticsearch.yml"));
-        Files.createDirectories(home.resolve("plugins"));
+        Path plugins = Files.createDirectories(home.resolve("plugins"));
+        assertTrue(Files.exists(plugins));
         Settings settings = Settings.builder()
             .put("path.home", home)
             .build();
         return new Environment(settings);
     }
 
+    static Path createPluginDir(Function<String, Path> temp) throws IOException {
+        return temp.apply("pluginDir");
+    }
+
     /** creates a fake jar file with empty class files */
     static void writeJar(Path jar, String... classes) throws IOException {
         try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(jar))) {
@@ -118,15 +186,40 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     static MockTerminal installPlugin(String pluginUrl, Environment env) throws Exception {
+        return installPlugin(pluginUrl, env, false);
+    }
+
+    static MockTerminal installPlugin(String pluginUrl, Environment env, boolean jarHellCheck) throws Exception {
         MockTerminal terminal = new MockTerminal();
-        CliTool.ExitStatus status = new InstallPluginCommand(terminal, pluginUrl, true).execute(env.settings(), env);
-        assertEquals(CliTool.ExitStatus.OK, status);
+        new InstallPluginCommand(env) {
+            @Override
+            void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {
+                if (jarHellCheck) {
+                    super.jarHellCheck(candidate, pluginsDir);
+                }
+            }
+        }.execute(terminal, pluginUrl, true);
         return terminal;
     }
 
     void assertPlugin(String name, Path original, Environment env) throws IOException {
         Path got = env.pluginsFile().resolve(name);
         assertTrue("dir " + name + " exists", Files.exists(got));
+
+        if (isPosix) {
+            Set<PosixFilePermission> perms = Files.getPosixFilePermissions(got);
+            assertThat(
+                perms,
+                containsInAnyOrder(
+                    PosixFilePermission.OWNER_READ,
+                    PosixFilePermission.OWNER_WRITE,
+                    PosixFilePermission.OWNER_EXECUTE,
+                    PosixFilePermission.GROUP_READ,
+                    PosixFilePermission.GROUP_EXECUTE,
+                    PosixFilePermission.OTHERS_READ,
+                    PosixFilePermission.OTHERS_EXECUTE));
+        }
+
         assertTrue("jar was copied", Files.exists(got.resolve("plugin.jar")));
         assertFalse("bin was not copied", Files.exists(got.resolve("bin")));
         assertFalse("config was not copied", Files.exists(got.resolve("config")));
@@ -156,6 +249,16 @@ public class InstallPluginCommandTests extends ESTestCase {
             Path configDir = env.configFile().resolve(name);
             assertTrue("config dir exists", Files.exists(configDir));
             assertTrue("config is a dir", Files.isDirectory(configDir));
+
+            if (isPosix) {
+                Path configRoot = env.configFile();
+                PosixFileAttributes configAttributes =
+                    Files.getFileAttributeView(configRoot, PosixFileAttributeView.class).readAttributes();
+                PosixFileAttributes attributes = Files.getFileAttributeView(configDir, PosixFileAttributeView.class).readAttributes();
+                assertThat(attributes.owner(), equalTo(configAttributes.owner()));
+                assertThat(attributes.group(), equalTo(configAttributes.group()));
+            }
+
             try (DirectoryStream<Path> stream = Files.newDirectoryStream(configDir)) {
                 for (Path file : stream) {
                     assertFalse("not a dir", Files.isDirectory(file));
@@ -176,16 +279,16 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     public void testSomethingWorks() throws Exception {
-        Environment env = createEnv();
-        Path pluginDir = createTempDir();
+        Environment env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
         String pluginZip = createPlugin("fake", pluginDir);
         installPlugin(pluginZip, env);
         assertPlugin("fake", pluginDir, env);
     }
 
     public void testSpaceInUrl() throws Exception {
-        Environment env = createEnv();
-        Path pluginDir = createTempDir();
+        Environment env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
         String pluginZip = createPlugin("fake", pluginDir);
         Path pluginZipWithSpaces = createTempFile("foo bar", ".zip");
         try (InputStream in = new URL(pluginZip).openStream()) {
@@ -196,28 +299,30 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     public void testMalformedUrlNotMaven() throws Exception {
+        Environment env = createEnv(fs, temp);
         // has two colons, so it appears similar to maven coordinates
         MalformedURLException e = expectThrows(MalformedURLException.class, () -> {
-            installPlugin("://host:1234", createEnv());
+            installPlugin("://host:1234", env);
         });
         assertTrue(e.getMessage(), e.getMessage().contains("no protocol"));
     }
 
     public void testPluginsDirMissing() throws Exception {
-        Environment env = createEnv();
+        Environment env = createEnv(fs, temp);
         Files.delete(env.pluginsFile());
-        Path pluginDir = createTempDir();
+        Path pluginDir = createPluginDir(temp);
         String pluginZip = createPlugin("fake", pluginDir);
         installPlugin(pluginZip, env);
         assertPlugin("fake", pluginDir, env);
     }
 
     public void testPluginsDirReadOnly() throws Exception {
-        assumeTrue("posix filesystem", isPosix);
-        Environment env = createEnv();
+        assumeTrue("posix and filesystem", isPosix && isReal);
+        Environment env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
         try (PosixPermissionsResetter pluginsAttrs = new PosixPermissionsResetter(env.pluginsFile())) {
             pluginsAttrs.setPermissions(new HashSet<>());
-            String pluginZip = createPlugin("fake", createTempDir());
+            String pluginZip = createPlugin("fake", pluginDir);
             IOException e = expectThrows(IOException.class, () -> {
                 installPlugin(pluginZip, env);
             });
@@ -227,8 +332,9 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     public void testBuiltinModule() throws Exception {
-        Environment env = createEnv();
-        String pluginZip = createPlugin("lang-groovy", createTempDir());
+        Environment env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        String pluginZip = createPlugin("lang-groovy", pluginDir);
         UserError e = expectThrows(UserError.class, () -> {
             installPlugin(pluginZip, env);
         });
@@ -237,66 +343,36 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     public void testJarHell() throws Exception {
-        Environment env = createEnv();
-        Path pluginDir = createTempDir();
-        writeJar(pluginDir.resolve("other.jar"), "FakePlugin");
-        String pluginZip = createPlugin("fake", pluginDir); // adds plugin.jar with FakePlugin
+        // jar hell test needs a real filesystem
+        assumeTrue("real filesystem", isReal);
+        Environment environment = createEnv(fs, temp);
+        Path pluginDirectory = createPluginDir(temp);
+        writeJar(pluginDirectory.resolve("other.jar"), "FakePlugin");
+        String pluginZip = createPlugin("fake", pluginDirectory); // adds plugin.jar with FakePlugin
         IllegalStateException e = expectThrows(IllegalStateException.class, () -> {
-            installPlugin(pluginZip, env);
+            installPlugin(pluginZip, environment, true);
         });
         assertTrue(e.getMessage(), e.getMessage().contains("jar hell"));
-        assertInstallCleaned(env);
+        assertInstallCleaned(environment);
     }
 
     public void testIsolatedPlugins() throws Exception {
-        Environment env = createEnv();
+        Environment env = createEnv(fs, temp);
         // these both share the same FakePlugin class
-        Path pluginDir1 = createTempDir();
+        Path pluginDir1 = createPluginDir(temp);
         String pluginZip1 = createPlugin("fake1", pluginDir1);
         installPlugin(pluginZip1, env);
-        Path pluginDir2 = createTempDir();
+        Path pluginDir2 = createPluginDir(temp);
         String pluginZip2 = createPlugin("fake2", pluginDir2);
         installPlugin(pluginZip2, env);
         assertPlugin("fake1", pluginDir1, env);
         assertPlugin("fake2", pluginDir2, env);
     }
 
-    public void testPurgatoryJarHell() throws Exception {
-        Environment env = createEnv();
-        Path pluginDir1 = createTempDir();
-        PluginTestUtil.writeProperties(pluginDir1,
-            "description", "fake desc",
-            "name", "fake1",
-            "version", "1.0",
-            "elasticsearch.version", Version.CURRENT.toString(),
-            "java.version", System.getProperty("java.specification.version"),
-            "classname", "FakePlugin",
-            "isolated", "false");
-        writeJar(pluginDir1.resolve("plugin.jar"), "FakePlugin");
-        String pluginZip1 = writeZip(pluginDir1, "elasticsearch");
-        installPlugin(pluginZip1, env);
-
-        Path pluginDir2 = createTempDir();
-        PluginTestUtil.writeProperties(pluginDir2,
-            "description", "fake desc",
-            "name", "fake2",
-            "version", "1.0",
-            "elasticsearch.version", Version.CURRENT.toString(),
-            "java.version", System.getProperty("java.specification.version"),
-            "classname", "FakePlugin",
-            "isolated", "false");
-        writeJar(pluginDir2.resolve("plugin.jar"), "FakePlugin");
-        String pluginZip2 = writeZip(pluginDir2, "elasticsearch");
-        IllegalStateException e = expectThrows(IllegalStateException.class, () -> {
-            installPlugin(pluginZip2, env);
-        });
-        assertTrue(e.getMessage(), e.getMessage().contains("jar hell"));
-        assertInstallCleaned(env);
-    }
-
     public void testExistingPlugin() throws Exception {
-        Environment env = createEnv();
-        String pluginZip = createPlugin("fake", createTempDir());
+        Environment env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        String pluginZip = createPlugin("fake", pluginDir);
         installPlugin(pluginZip, env);
         UserError e = expectThrows(UserError.class, () -> {
             installPlugin(pluginZip, env);
@@ -306,8 +382,8 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     public void testBin() throws Exception {
-        Environment env = createEnv();
-        Path pluginDir = createTempDir();
+        Environment env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
         Path binDir = pluginDir.resolve("bin");
         Files.createDirectory(binDir);
         Files.createFile(binDir.resolve("somescript"));
@@ -317,8 +393,8 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     public void testBinNotDir() throws Exception {
-        Environment env = createEnv();
-        Path pluginDir = createTempDir();
+        Environment env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
         Path binDir = pluginDir.resolve("bin");
         Files.createFile(binDir);
         String pluginZip = createPlugin("fake", pluginDir);
@@ -330,8 +406,8 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     public void testBinContainsDir() throws Exception {
-        Environment env = createEnv();
-        Path pluginDir = createTempDir();
+        Environment env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
         Path dirInBinDir = pluginDir.resolve("bin").resolve("foo");
         Files.createDirectories(dirInBinDir);
         Files.createFile(dirInBinDir.resolve("somescript"));
@@ -344,8 +420,8 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     public void testBinConflict() throws Exception {
-        Environment env = createEnv();
-        Path pluginDir = createTempDir();
+        Environment env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
         Path binDir = pluginDir.resolve("bin");
         Files.createDirectory(binDir);
         Files.createFile(binDir.resolve("somescript"));
@@ -359,8 +435,8 @@ public class InstallPluginCommandTests extends ESTestCase {
 
     public void testBinPermissions() throws Exception {
         assumeTrue("posix filesystem", isPosix);
-        Environment env = createEnv();
-        Path pluginDir = createTempDir();
+        Environment env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
         Path binDir = pluginDir.resolve("bin");
         Files.createDirectory(binDir);
         Files.createFile(binDir.resolve("somescript"));
@@ -376,8 +452,8 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     public void testConfig() throws Exception {
-        Environment env = createEnv();
-        Path pluginDir = createTempDir();
+        Environment env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
         Path configDir = pluginDir.resolve("config");
         Files.createDirectory(configDir);
         Files.createFile(configDir.resolve("custom.yaml"));
@@ -387,11 +463,11 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     public void testExistingConfig() throws Exception {
-        Environment env = createEnv();
+        Environment env = createEnv(fs, temp);
         Path envConfigDir = env.configFile().resolve("fake");
         Files.createDirectories(envConfigDir);
         Files.write(envConfigDir.resolve("custom.yaml"), "existing config".getBytes(StandardCharsets.UTF_8));
-        Path pluginDir = createTempDir();
+        Path pluginDir = createPluginDir(temp);
         Path configDir = pluginDir.resolve("config");
         Files.createDirectory(configDir);
         Files.write(configDir.resolve("custom.yaml"), "new config".getBytes(StandardCharsets.UTF_8));
@@ -406,8 +482,8 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     public void testConfigNotDir() throws Exception {
-        Environment env = createEnv();
-        Path pluginDir = createTempDir();
+        Environment env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
         Path configDir = pluginDir.resolve("config");
         Files.createFile(configDir);
         String pluginZip = createPlugin("fake", pluginDir);
@@ -419,8 +495,8 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     public void testConfigContainsDir() throws Exception {
-        Environment env = createEnv();
-        Path pluginDir = createTempDir();
+        Environment env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
         Path dirInConfigDir = pluginDir.resolve("config").resolve("foo");
         Files.createDirectories(dirInConfigDir);
         Files.createFile(dirInConfigDir.resolve("myconfig.yml"));
@@ -433,8 +509,8 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     public void testConfigConflict() throws Exception {
-        Environment env = createEnv();
-        Path pluginDir = createTempDir();
+        Environment env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
         Path configDir = pluginDir.resolve("config");
         Files.createDirectory(configDir);
         Files.createFile(configDir.resolve("myconfig.yml"));
@@ -447,8 +523,8 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     public void testMissingDescriptor() throws Exception {
-        Environment env = createEnv();
-        Path pluginDir = createTempDir();
+        Environment env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
         Files.createFile(pluginDir.resolve("fake.yml"));
         String pluginZip = writeZip(pluginDir, "elasticsearch");
         NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> {
@@ -459,8 +535,8 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     public void testMissingDirectory() throws Exception {
-        Environment env = createEnv();
-        Path pluginDir = createTempDir();
+        Environment env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
         Files.createFile(pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES));
         String pluginZip = writeZip(pluginDir, null);
         UserError e = expectThrows(UserError.class, () -> {
@@ -470,6 +546,19 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertInstallCleaned(env);
     }
 
+    public void testZipRelativeOutsideEntryName() throws Exception {
+        Environment env = createEnv(fs, temp);
+        Path zip = createTempDir().resolve("broken.zip");
+        try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(zip))) {
+            stream.putNextEntry(new ZipEntry("elasticsearch/../blah"));
+        }
+        String pluginZip = zip.toUri().toURL().toString();
+        IOException e = expectThrows(IOException.class, () -> {
+            installPlugin(pluginZip, env);
+        });
+        assertTrue(e.getMessage(), e.getMessage().contains("resolving outside of plugin directory"));
+    }
+
     // TODO: test batch flag?
     // TODO: test checksum (need maven/official below)
     // TODO: test maven, official, and staging install...need tests with fixtures...
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java
index c86a6464eb0..cbdd031dea1 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java
@@ -22,14 +22,10 @@ package org.elasticsearch.plugins;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Collections;
-import java.util.List;
 
 import org.apache.lucene.util.LuceneTestCase;
-import org.elasticsearch.common.cli.CliTool;
-import org.elasticsearch.common.cli.CliToolTestCase;
-import org.elasticsearch.common.cli.MockTerminal;
-import org.elasticsearch.common.cli.Terminal;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
@@ -48,8 +44,9 @@ public class ListPluginsCommandTests extends ESTestCase {
 
     static MockTerminal listPlugins(Environment env) throws Exception {
         MockTerminal terminal = new MockTerminal();
-        CliTool.ExitStatus status = new ListPluginsCommand(terminal).execute(env.settings(), env);
-        assertEquals(CliTool.ExitStatus.OK, status);
+        String[] args = {};
+        int status = new ListPluginsCommand(env).main(args, terminal);
+        assertEquals(ExitCodes.OK, status);
         return terminal;
     }
 
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java
index acc300c6cf5..466f7d05cd1 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.plugins;
 
-import org.elasticsearch.common.cli.Terminal;
+import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.test.ESTestCase;
 
 import java.nio.file.Path;
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
index 0bfdf5c34a8..d9d5661b834 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
@@ -25,11 +25,8 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 
 import org.apache.lucene.util.LuceneTestCase;
-import org.elasticsearch.common.cli.CliTool;
-import org.elasticsearch.common.cli.CliToolTestCase;
-import org.elasticsearch.common.cli.MockTerminal;
-import org.elasticsearch.common.cli.Terminal;
-import org.elasticsearch.common.cli.UserError;
+import org.elasticsearch.cli.UserError;
+import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
@@ -51,8 +48,7 @@ public class RemovePluginCommandTests extends ESTestCase {
 
     static MockTerminal removePlugin(String name, Environment env) throws Exception {
         MockTerminal terminal = new MockTerminal();
-        CliTool.ExitStatus status = new RemovePluginCommand(terminal, name).execute(env.settings(), env);
-        assertEquals(CliTool.ExitStatus.OK, status);
+        new RemovePluginCommand(env).execute(terminal, name);
         return terminal;
     }
 
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java
index 9c68ea196aa..63c09890acc 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java
@@ -23,7 +23,7 @@ import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.service.InternalClusterService;
+import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
@@ -66,14 +66,14 @@ public class TribeUnitTests extends ESTestCase {
                 .put(baseSettings)
                 .put("cluster.name", "tribe1")
                 .put("node.name", "tribe1_node")
-                    .put(InternalClusterService.NODE_ID_SEED_SETTING.getKey(), random().nextLong())
+                    .put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), random().nextLong())
                 .build()).start();
         tribe2 = new TribeClientNode(
             Settings.builder()
                 .put(baseSettings)
                 .put("cluster.name", "tribe2")
                 .put("node.name", "tribe2_node")
-                    .put(InternalClusterService.NODE_ID_SEED_SETTING.getKey(), random().nextLong())
+                    .put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), random().nextLong())
                 .build()).start();
     }
 
diff --git a/qa/smoke-test-ingest-disabled/build.gradle b/qa/smoke-test-ingest-disabled/build.gradle
index ca71697a7b4..09b2d1409a1 100644
--- a/qa/smoke-test-ingest-disabled/build.gradle
+++ b/qa/smoke-test-ingest-disabled/build.gradle
@@ -21,6 +21,6 @@ apply plugin: 'elasticsearch.rest-test'
 
 integTest {
     cluster {
-        systemProperty 'es.node.ingest', 'false'
+        setting 'node.ingest', 'false'
     }
 }
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yaml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yaml
index 491e1dae292..27114ac61f2 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yaml
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yaml
@@ -219,3 +219,110 @@
   - length: { _source: 2 }
   - match: { _source.do_nothing: "foo" }
   - match: { _source.error: "processor first_processor [remove]: field [field_to_remove] not present as part of path [field_to_remove]" }
+
+---
+"Test rolling up json object arrays":
+  - do:
+      ingest.put_pipeline:
+        id: "_id"
+        body:  >
+          {
+            "processors": [
+               {
+                  "foreach": {
+                     "field": "values",
+                     "processors": [
+                        {
+                          "append": {
+                             "field": "values_flat",
+                             "value": "{{_value.key}}_{{_value.value}}"
+                          }
+                       }
+                     ]
+                  }
+               }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "_id"
+        body: {
+          values_flat : [],
+          values: [
+             {
+                level: 1,
+                key: "foo",
+                value: "bar"
+             },
+             {
+                level: 2,
+                key: "foo",
+                value: "baz"
+             }
+          ]
+        }
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - length: { _source: 2 }
+  - match: { _source.values_flat: ["foo_bar", "foo_baz"] }
+
+---
+"Test verbose simulate with error context":
+  - do:
+      cluster.health:
+          wait_for_status: green
+  - do:
+      ingest.simulate:
+        verbose: true
+        body: >
+          {
+            "pipeline" : {
+              "description": "_description",
+              "processors": [
+                {
+                  "rename" : {
+                    "tag" : "rename-status",
+                    "field" : "status",
+                    "to" : "bar",
+                    "on_failure" : [
+                      {
+                        "set" : {
+                          "tag" : "set_on_rename_failure",
+                          "field" : "error",
+                          "value" : "processor {{ _ingest.on_failure_processor_tag }} [{{ _ingest.on_failure_processor_type }}]: {{ _ingest.on_failure_message }}"
+                        }
+                      }
+                    ]
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_index": "index",
+                "_type": "type",
+                "_id": "id",
+                "_source": {
+                  "foo": "bar"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - length: { docs.0.processor_results: 2 }
+  - match: { docs.0.processor_results.0.tag: "rename-status" }
+  - match: { docs.0.processor_results.0.error.type: "illegal_argument_exception" }
+  - match: { docs.0.processor_results.0.error.reason: "field [status] doesn't exist" }
+  - match: { docs.0.processor_results.1.tag: "set_on_rename_failure" }
+  - length: { docs.0.processor_results.1.doc._source: 2 }
+  - match: { docs.0.processor_results.1.doc._source.foo: "bar"  }
+  - match: { docs.0.processor_results.1.doc._source.error: "processor rename-status [rename]: field [status] doesn't exist" }
diff --git a/qa/smoke-test-reindex-with-groovy/build.gradle b/qa/smoke-test-reindex-with-groovy/build.gradle
index a42f5e708a2..c4b462ce45a 100644
--- a/qa/smoke-test-reindex-with-groovy/build.gradle
+++ b/qa/smoke-test-reindex-with-groovy/build.gradle
@@ -21,6 +21,6 @@ apply plugin: 'elasticsearch.rest-test'
 
 integTest {
   cluster {
-    systemProperty 'es.script.inline', 'true'
+    setting 'script.inline', 'true'
   }
 }
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/10_script.yaml b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/10_script.yaml
index d37a94deea7..aa553a5c9dc 100644
--- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/10_script.yaml
+++ b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/10_script.yaml
@@ -395,3 +395,51 @@
             match:
               user: otherkimchy
   - match: { hits.total: 1 }
+
+---
+"Change index to write to a different index":
+  - do:
+      index:
+        index:        twitter
+        type:         tweet
+        id:           1
+        body:         { "user": "kimchy" }
+  - do:
+      index:
+        index:        twitter
+        type:         tweet
+        id:           2
+        body:         { "user": "another" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            index: twitter
+          dest:
+            index: new_twitter
+          script:
+            inline: if (ctx._source.user == "kimchy") {ctx._index = 'other_new_twitter'}
+  - match: {created: 2}
+  - match: {noops: 0}
+
+  - do:
+      search:
+        index: other_new_twitter
+        body:
+          query:
+            match:
+              user: kimchy
+  - match: { hits.total: 1 }
+
+  - do:
+      search:
+        index: new_twitter
+        body:
+          query:
+            match:
+              user: another
+  - match: { hits.total: 1 }
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml
new file mode 100644
index 00000000000..ddd22246717
--- /dev/null
+++ b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml
@@ -0,0 +1,29 @@
+---
+"Timeout":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      catch: request_timeout
+      reindex:
+        refresh: true
+        body:
+          source:
+            index: twitter
+            timeout: 10
+            query:
+              script:
+                # Sleep 100x longer than the timeout. That should cause a timeout!
+                # Return true causes the document to try to be collected which is what actually triggers the timeout.
+                script: sleep(1000); return true
+          dest:
+            index: new_twitter
+  - is_true: timed_out
+  - match: {created: 0}
+  - match: {noops: 0}
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml
new file mode 100644
index 00000000000..50442c2d51b
--- /dev/null
+++ b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml
@@ -0,0 +1,34 @@
+---
+"Response format search failures":
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      catch: request
+      reindex:
+        body:
+          source:
+            index:   source
+            query:
+              script:
+                script: 1/0 # Divide by 0 to cause a search time exception
+          dest:
+            index:   dest
+  - match: {created: 0}
+  - match: {updated: 0}
+  - match: {version_conflicts: 0}
+  - match: {batches: 0}
+  - is_true: failures.0.shard
+  - match: {failures.0.index:  source}
+  - is_true: failures.0.node
+  - match: {failures.0.reason.type:   script_exception}
+  - match: {failures.0.reason.reason: "failed to run inline script [1/0] using lang [groovy]"}
+  - match: {failures.0.reason.caused_by.type:   arithmetic_exception}
+  - match: {failures.0.reason.caused_by.reason: Division by zero}
+  - is_true: took
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update-by-query/30_timeout.yaml b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update-by-query/30_timeout.yaml
new file mode 100644
index 00000000000..2a291bf0541
--- /dev/null
+++ b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update-by-query/30_timeout.yaml
@@ -0,0 +1,26 @@
+---
+"Timeout":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      catch: request_timeout
+      update-by-query:
+        index:   twitter
+        refresh: true
+        search_timeout: 10ms
+        body:
+          query:
+            script:
+              # Sleep 100x longer than the timeout. That should cause a timeout!
+              # Return true causes the document to try to be collected which is what actually triggers the timeout.
+              script: sleep(1000); return true
+  - is_true: timed_out
+  - match: {updated: 0}
+  - match: {noops: 0}
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update-by-query/40_search_failure.yaml b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update-by-query/40_search_failure.yaml
new file mode 100644
index 00000000000..8f89409c586
--- /dev/null
+++ b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update-by-query/40_search_failure.yaml
@@ -0,0 +1,30 @@
+---
+"Response format search failures":
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      catch: request
+      update-by-query:
+        index:   source
+        body:
+          query:
+            script:
+              script: 1/0 # Divide by 0 to cause a search time exception
+  - match: {updated: 0}
+  - match: {version_conflicts: 0}
+  - match: {batches: 0}
+  - is_true: failures.0.shard
+  - match: {failures.0.index:  source}
+  - is_true: failures.0.node
+  - match: {failures.0.reason.type:   script_exception}
+  - match: {failures.0.reason.reason: "failed to run inline script [1/0] using lang [groovy]"}
+  - match: {failures.0.reason.caused_by.type:   arithmetic_exception}
+  - match: {failures.0.reason.caused_by.reason: Division by zero}
+  - is_true: took
diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle
index d0be6f13946..1df5b7dc402 100644
--- a/qa/vagrant/build.gradle
+++ b/qa/vagrant/build.gradle
@@ -26,8 +26,17 @@ String testScripts = '*.bats'
 String testCommand = "cd \$TESTROOT && sudo bats --tap \$BATS/$testScripts"
 String smokeTestCommand = 'echo I work'
 List<String> representativeBoxes = ['ubuntu-1404', 'centos-7']
-List<String> boxes = representativeBoxes + ['ubuntu-1204', 'ubuntu-1504',
-  'debian-8', 'centos-6', 'oel-7', 'fedora-22', 'opensuse-13', 'sles-12']
+List<String> boxes = representativeBoxes + [
+        'ubuntu-1204',
+        'ubuntu-1504',
+        'debian-8',
+        'centos-6',
+        'oel-6',
+        'oel-7',
+        'fedora-22',
+        'opensuse-13',
+        'sles-12'
+]
 
 /* The version of elasticsearch that we upgrade *from* as part of testing
  * upgrades. */
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/25_tar_plugins.bats b/qa/vagrant/src/test/resources/packaging/scripts/25_tar_plugins.bats
index 8f55b1eb78c..552c404a3d6 120000
--- a/qa/vagrant/src/test/resources/packaging/scripts/25_tar_plugins.bats
+++ b/qa/vagrant/src/test/resources/packaging/scripts/25_tar_plugins.bats
@@ -1 +1 @@
-plugin_test_cases.bash
\ No newline at end of file
+module_and_plugin_test_cases.bash
\ No newline at end of file
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/50_modules_and_plugins.bats b/qa/vagrant/src/test/resources/packaging/scripts/50_modules_and_plugins.bats
new file mode 120000
index 00000000000..552c404a3d6
--- /dev/null
+++ b/qa/vagrant/src/test/resources/packaging/scripts/50_modules_and_plugins.bats
@@ -0,0 +1 @@
+module_and_plugin_test_cases.bash
\ No newline at end of file
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/50_plugins.bats b/qa/vagrant/src/test/resources/packaging/scripts/50_plugins.bats
deleted file mode 120000
index 8f55b1eb78c..00000000000
--- a/qa/vagrant/src/test/resources/packaging/scripts/50_plugins.bats
+++ /dev/null
@@ -1 +0,0 @@
-plugin_test_cases.bash
\ No newline at end of file
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats b/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats
index 65d212e56d8..48bf2aca4e3 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats
+++ b/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats
@@ -83,7 +83,8 @@ setup() {
 }
 
 @test "[UPGRADE] start version under test" {
-    start_elasticsearch_service yellow
+    start_elasticsearch_service yellow library
+    wait_for_elasticsearch_status yellow library2
 }
 
 @test "[UPGRADE] check elasticsearch version is version under test" {
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/90_reinstall.bats b/qa/vagrant/src/test/resources/packaging/scripts/90_reinstall.bats
new file mode 100644
index 00000000000..3c2f7be7330
--- /dev/null
+++ b/qa/vagrant/src/test/resources/packaging/scripts/90_reinstall.bats
@@ -0,0 +1,72 @@
+#!/usr/bin/env bats
+
+# Tests upgrading elasticsearch from a previous version with the deb or rpm
+# packages. Just uses a single node cluster on the current machine rather than
+# fancy rolling restarts.
+
+# WARNING: This testing file must be executed as root and can
+# dramatically change your system. It removes the 'elasticsearch'
+# user/group and also many directories. Do not execute this file
+# unless you know exactly what you are doing.
+
+# The test case can be executed with the Bash Automated
+# Testing System tool available at https://github.com/sstephenson/bats
+# Thanks to Sam Stephenson!
+
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Load test utilities
+load packaging_test_utils
+load os_package
+
+# Cleans everything for the 1st execution
+setup() {
+    skip_not_dpkg_or_rpm
+}
+
+@test "[REINSTALL] install" {
+    clean_before_test
+    install_package
+}
+
+@test "[REINSTALL] purge elasticsearch" {
+    purge_elasticsearch
+}
+
+@test "[REINSTALL] chown directories" {
+    # to simulate the loss of ownership
+    if [ -d /var/lib/elasticsearch ]; then
+      sudo chown -R root:root /var/lib/elasticsearch
+    fi
+    if [ -d "/var/log/elasticsearch" ]; then
+      sudo chown -R root:root /var/log/elasticsearch
+    fi
+    if [ -d /etc/elasticsearch ]; then
+      sudo chown -R root:root /etc/elasticsearch
+    fi
+}
+
+@test "[REINSTALL] reinstall elasticsearch" {
+    install_package
+}
+
+@test "[REINSTALL] check ownership" {
+    assert_recursive_ownership /var/lib/elasticsearch elasticsearch elasticsearch
+    assert_recursive_ownership /var/log/elasticsearch elasticsearch elasticsearch
+    assert_recursive_ownership /etc/elasticsearch root elasticsearch
+}
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
similarity index 81%
rename from qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash
rename to qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
index c81d850d94d..e92c4b62b76 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
@@ -33,13 +33,14 @@
 ##################################
 # Common test cases for both tar and rpm/deb based plugin tests
 ##################################
-# This file is symlinked to both 25_tar_plugins.bats and 50_plugins.bats so its
+# This file is symlinked to both 25_tar_plugins.bats and 50_modules_and_plugins.bats so its
 # executed twice - once to test plugins using the tar distribution and once to
 # test files using the rpm distribution or the deb distribution, whichever the
 # system uses.
 
 # Load test utilities
 load packaging_test_utils
+load modules
 load plugins
 
 setup() {
@@ -102,7 +103,7 @@ fi
         echo "CONF_FILE=$CONF_FILE" >> /etc/sysconfig/elasticsearch;
     fi
 
-    run_elasticsearch_service 1 -Des.default.config="$CONF_FILE"
+    run_elasticsearch_service 1 -Ees.default.config="$CONF_FILE"
 
     # remove settings again otherwise cleaning up before next testrun will fail
     if is_dpkg ; then
@@ -219,16 +220,41 @@ fi
     install_and_check_plugin discovery ec2 aws-java-sdk-core-*.jar
 }
 
-@test "[$GROUP] install lang-expression plugin" {
-    install_and_check_plugin lang expression
+@test "[$GROUP] install ingest-attachment plugin" {
+    # we specify the version on the poi-3.13.jar so that the test does
+    # not spuriously pass if the jar is missing but the other poi jars
+    # are present
+    install_and_check_plugin ingest attachment bcprov-jdk15on-*.jar tika-core-*.jar pdfbox-*.jar poi-3.13.jar
 }
 
-@test "[$GROUP] install lang-groovy plugin" {
-    install_and_check_plugin lang groovy
+@test "[$GROUP] install ingest-geoip plugin" {
+    install_and_check_plugin ingest geoip geoip2-*.jar jackson-annotations-*.jar jackson-databind-*.jar maxmind-db-*.jar
 }
 
-@test "[$GROUP] install lang-painless plugin" {
-    install_and_check_plugin lang painless
+@test "[$GROUP] check ingest-grok module" {
+    check_module ingest-grok jcodings-*.jar joni-*.jar
+}
+
+@test "[$GROUP] check lang-expression module" {
+    # we specify the version on the asm-5.0.4.jar so that the test does
+    # not spuriously pass if the jar is missing but the other asm jars
+    # are present
+    check_secure_module lang-expression antlr4-runtime-*.jar asm-5.0.4.jar asm-commons-*.jar asm-tree-*.jar lucene-expressions-*.jar
+}
+
+@test "[$GROUP] check lang-groovy module" {
+    check_secure_module lang-groovy groovy-*-indy.jar
+}
+
+@test "[$GROUP] check lang-mustache module" {
+    check_secure_module lang-mustache compiler-*.jar
+}
+
+@test "[$GROUP] check lang-painless module" {
+    # we specify the version on the asm-5.0.4.jar so that the test does
+    # not spuriously pass if the jar is missing but the other asm jars
+    # are present
+    check_secure_module lang-painless antlr4-runtime-*.jar asm-5.0.4.jar asm-commons-*.jar asm-tree-*.jar
 }
 
 @test "[$GROUP] install javascript plugin" {
@@ -247,6 +273,14 @@ fi
     install_and_check_plugin mapper murmur3
 }
 
+@test "[$GROUP] check reindex module" {
+    check_module reindex
+}
+
+@test "[$GROUP] install repository-hdfs plugin" {
+    install_and_check_plugin repository hdfs hadoop-client-*.jar hadoop-common-*.jar hadoop-annotations-*.jar hadoop-auth-*.jar hadoop-hdfs-*.jar htrace-core-*.jar guava-*.jar protobuf-java-*.jar commons-logging-*.jar commons-cli-*.jar commons-collections-*.jar commons-configuration-*.jar commons-io-*.jar commons-lang-*.jar servlet-api-*.jar slf4j-api-*.jar
+}
+
 @test "[$GROUP] install size mapper plugin" {
     install_and_check_plugin mapper size
 }
@@ -264,7 +298,7 @@ fi
 }
 
 @test "[$GROUP] check the installed plugins can be listed with 'plugins list' and result matches the list of plugins in plugins pom" {
-    "$ESHOME/bin/elasticsearch-plugin" list | tail -n +2 | sed 's/^......//' > /tmp/installed
+    "$ESHOME/bin/elasticsearch-plugin" list > /tmp/installed
     compare_plugins_list "/tmp/installed" "'plugins list'"
 }
 
@@ -321,16 +355,12 @@ fi
     remove_plugin discovery-ec2
 }
 
-@test "[$GROUP] remove lang-expression plugin" {
-    remove_plugin lang-expression
+@test "[$GROUP] remove ingest-attachment plugin" {
+    remove_plugin ingest-attachment
 }
 
-@test "[$GROUP] remove lang-groovy plugin" {
-    remove_plugin lang-groovy
-}
-
-@test "[$GROUP] remove lang-painless plugin" {
-    remove_plugin lang-painless
+@test "[$GROUP] remove ingest-geoip plugin" {
+    remove_plugin ingest-geoip
 }
 
 @test "[$GROUP] remove javascript plugin" {
@@ -357,12 +387,12 @@ fi
     remove_plugin repository-azure
 }
 
-@test "[$GROUP] remove repository-s3 plugin" {
-    remove_plugin repository-s3
+@test "[$GROUP] remove repository-hdfs plugin" {
+    remove_plugin repository-hdfs
 }
 
-@test "[$GROUP] remove site example plugin" {
-    remove_plugin site-example
+@test "[$GROUP] remove repository-s3 plugin" {
+    remove_plugin repository-s3
 }
 
 @test "[$GROUP] remove store-smb plugin" {
@@ -393,14 +423,14 @@ fi
     local loglines=$(cat /tmp/plugin-cli-output | wc -l)
     if [ "$GROUP" == "TAR PLUGINS" ]; then
     # tar extraction does not create the plugins directory so the plugin tool will print an additional line that the directory will be created
-        [ "$loglines" -eq "7" ] || {
-            echo "Expected 7 lines but the output was:"
+        [ "$loglines" -eq "3" ] || {
+            echo "Expected 3 lines but the output was:"
             cat /tmp/plugin-cli-output
             false
         }
     else
-        [ "$loglines" -eq "6" ] || {
-            echo "Expected 6 lines but the output was:"
+        [ "$loglines" -eq "2" ] || {
+            echo "Expected 2 lines but the output was:"
             cat /tmp/plugin-cli-output
             false
         }
@@ -411,14 +441,14 @@ fi
     sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" install "file://$relativePath" -Des.logger.level=DEBUG > /tmp/plugin-cli-output
     local loglines=$(cat /tmp/plugin-cli-output | wc -l)
     if [ "$GROUP" == "TAR PLUGINS" ]; then
-        [ "$loglines" -gt "7" ] || {
-            echo "Expected more than 7 lines but the output was:"
+        [ "$loglines" -gt "3" ] || {
+            echo "Expected more than 3 lines but the output was:"
             cat /tmp/plugin-cli-output
             false
         }
     else
-        [ "$loglines" -gt "6" ] || {
-            echo "Expected more than 6 lines but the output was:"
+        [ "$loglines" -gt "2" ] || {
+            echo "Expected more than 2 lines but the output was:"
             cat /tmp/plugin-cli-output
             false
         }
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/modules.bash b/qa/vagrant/src/test/resources/packaging/scripts/modules.bash
new file mode 100644
index 00000000000..bd6da680da9
--- /dev/null
+++ b/qa/vagrant/src/test/resources/packaging/scripts/modules.bash
@@ -0,0 +1,44 @@
+#!/bin/sh
+
+# This file contains some utilities to test the elasticsearch scripts,
+# the .deb/.rpm packages and the SysV/Systemd scripts.
+
+# WARNING: This testing file must be executed as root and can
+# dramatically change your system. It removes the 'elasticsearch'
+# user/group and also many directories. Do not execute this file
+# unless you know exactly what you are doing.
+
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+check_module() {
+    local name=$1
+    shift
+
+    assert_module_or_plugin_directory "$ESMODULES/$name"
+
+    for file in "$@"; do
+        assert_module_or_plugin_file "$ESMODULES/$name/$file"
+    done
+
+    assert_module_or_plugin_file "$ESMODULES/$name/$name-*.jar"
+    assert_module_or_plugin_file "$ESMODULES/$name/plugin-descriptor.properties"
+}
+
+check_secure_module() {
+    check_module "$@" plugin-security.policy
+}
\ No newline at end of file
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/os_package.bash b/qa/vagrant/src/test/resources/packaging/scripts/os_package.bash
index f48532cb3f3..72c59c39324 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/os_package.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/os_package.bash
@@ -30,6 +30,7 @@
 export_elasticsearch_paths() {
     export ESHOME="/usr/share/elasticsearch"
     export ESPLUGINS="$ESHOME/plugins"
+    export ESMODULES="$ESHOME/modules"
     export ESCONFIG="/etc/elasticsearch"
     export ESSCRIPTS="$ESCONFIG/scripts"
     export ESDATA="/var/lib/elasticsearch"
@@ -47,6 +48,7 @@ install_package() {
         case $opt in
             u)
                 rpmCommand='-U'
+                dpkgCommand='--force-confnew'
                 ;;
             v)
                 version=$OPTARG
@@ -59,7 +61,7 @@ install_package() {
     if is_rpm; then
         rpm $rpmCommand elasticsearch-$version.rpm
     elif is_dpkg; then
-        dpkg -i elasticsearch-$version.deb
+        dpkg $dpkgCommand -i elasticsearch-$version.deb
     else
         skip "Only rpm or deb supported"
     fi
@@ -81,7 +83,8 @@ verify_package_installation() {
     assert_file "$ESSCRIPTS" d root elasticsearch 750
     assert_file "$ESDATA" d elasticsearch elasticsearch 755
     assert_file "$ESLOG" d elasticsearch elasticsearch 755
-    assert_file "$ESPLUGINS" d elasticsearch elasticsearch 755
+    assert_file "$ESPLUGINS" d root root 755
+    assert_file "$ESMODULES" d root root 755
     assert_file "$ESPIDDIR" d elasticsearch elasticsearch 755
     assert_file "$ESHOME/NOTICE.txt" f root root 644
     assert_file "$ESHOME/README.textile" f root root 644
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash
index 11961e06921..133b6e16f1a 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash
@@ -179,10 +179,47 @@ assert_file() {
     fi
 }
 
+assert_module_or_plugin_directory() {
+    local directory=$1
+    shift
+
+    #owner group and permissions vary depending on how es was installed
+    #just make sure that everything is the same as $CONFIG_DIR, which was properly set up during install
+    config_user=$(find "$ESHOME" -maxdepth 0 -printf "%u")
+    config_owner=$(find "$ESHOME" -maxdepth 0 -printf "%g")
+    # directories should use the user file-creation mask
+    config_privileges=$(executable_privileges_for_user_from_umask $ESPLUGIN_COMMAND_USER)
+
+    assert_file $directory d $config_user $config_owner $(printf "%o" $config_privileges)
+}
+
+assert_module_or_plugin_file() {
+    local file=$1
+    shift
+
+    assert_file_exist "$(readlink -m $file)"
+
+    # config files should not be executable and otherwise use the user
+    # file-creation mask
+    expected_file_privileges=$(file_privileges_for_user_from_umask $ESPLUGIN_COMMAND_USER)
+    assert_file $file f $config_user $config_owner $(printf "%o" $expected_file_privileges)
+}
+
 assert_output() {
     echo "$output" | grep -E "$1"
 }
 
+assert_recursive_ownership() {
+    local directory=$1
+    local user=$2
+    local group=$3
+
+    realuser=$(find $directory -printf "%u\n" | sort | uniq)
+    [ "$realuser" = "$user" ]
+    realgroup=$(find $directory -printf "%g\n" | sort | uniq)
+    [ "$realgroup" = "$group" ]
+}
+
 # Deletes everything before running a test file
 clean_before_test() {
 
@@ -209,6 +246,22 @@ clean_before_test() {
     # Kills all running Elasticsearch processes
     ps aux | grep -i "org.elasticsearch.bootstrap.Elasticsearch" | awk {'print $2'} | xargs kill -9 > /dev/null 2>&1 || true
 
+    purge_elasticsearch
+
+    # Removes user & group
+    userdel elasticsearch > /dev/null 2>&1 || true
+    groupdel elasticsearch > /dev/null 2>&1 || true
+
+
+    # Removes all files
+    for d in "${ELASTICSEARCH_TEST_FILES[@]}"; do
+        if [ -e "$d" ]; then
+            rm -rf "$d"
+        fi
+    done
+}
+
+purge_elasticsearch() {
     # Removes RPM package
     if is_rpm; then
         rpm --quiet -e elasticsearch > /dev/null 2>&1 || true
@@ -226,28 +279,17 @@ clean_before_test() {
     if [ -x "`which apt-get 2>/dev/null`" ]; then
         apt-get --quiet --yes purge elasticsearch > /dev/null 2>&1 || true
     fi
-
-    # Removes user & group
-    userdel elasticsearch > /dev/null 2>&1 || true
-    groupdel elasticsearch > /dev/null 2>&1 || true
-
-
-    # Removes all files
-    for d in "${ELASTICSEARCH_TEST_FILES[@]}"; do
-        if [ -e "$d" ]; then
-            rm -rf "$d"
-        fi
-    done
 }
 
 # Start elasticsearch and wait for it to come up with a status.
 # $1 - expected status - defaults to green
 start_elasticsearch_service() {
     local desiredStatus=${1:-green}
+    local index=$2
 
     run_elasticsearch_service 0
 
-    wait_for_elasticsearch_status $desiredStatus
+    wait_for_elasticsearch_status $desiredStatus $index
 
     if [ -r "/tmp/elasticsearch/elasticsearch.pid" ]; then
         pid=$(cat /tmp/elasticsearch/elasticsearch.pid)
@@ -285,6 +327,9 @@ run_elasticsearch_service() {
     if [ -f "/tmp/elasticsearch/bin/elasticsearch" ]; then
         if [ -z "$CONF_DIR" ]; then
             local CONF_DIR=""
+            local ES_PATH_CONF=""
+        else
+            local ES_PATH_CONF="-Ees.path.conf=$CONF_DIR"
         fi
         # we must capture the exit code to compare so we don't want to start as background process in case we expect something other than 0
         local background=""
@@ -303,7 +348,7 @@ run_elasticsearch_service() {
 # This line is attempting to emulate the on login behavior of /usr/share/upstart/sessions/jayatana.conf
 [ -f /usr/share/java/jayatanaag.jar ] && export JAVA_TOOL_OPTIONS="-javaagent:/usr/share/java/jayatanaag.jar"
 # And now we can start Elasticsearch normally, in the background (-d) and with a pidfile (-p).
-$timeoutCommand/tmp/elasticsearch/bin/elasticsearch $background -p /tmp/elasticsearch/elasticsearch.pid -Des.path.conf=$CONF_DIR $commandLineArgs
+$timeoutCommand/tmp/elasticsearch/bin/elasticsearch $background -p /tmp/elasticsearch/elasticsearch.pid $ES_PATH_CONF $commandLineArgs
 BASH
         [ "$status" -eq "$expectedStatus" ]
     elif is_systemd; then
@@ -353,9 +398,10 @@ stop_elasticsearch_service() {
 # $1 - expected status - defaults to green
 wait_for_elasticsearch_status() {
     local desiredStatus=${1:-green}
+    local index=$2
 
     echo "Making sure elasticsearch is up..."
-    wget -O - --retry-connrefused --waitretry=1 --timeout=60 --tries 60 http://localhost:9200 || {
+    wget -O - --retry-connrefused --waitretry=1 --timeout=60 --tries 60 http://localhost:9200/_cluster/health || {
           echo "Looks like elasticsearch never started. Here is its log:"
           if [ -e "$ESLOG/elasticsearch.log" ]; then
               cat "$ESLOG/elasticsearch.log"
@@ -366,8 +412,13 @@ wait_for_elasticsearch_status() {
           false
     }
 
-    echo "Tring to connect to elasticsearch and wait for expected status..."
-    curl -sS "http://localhost:9200/_cluster/health?wait_for_status=$desiredStatus&timeout=60s&pretty"
+    if [ -z "index" ]; then
+      echo "Tring to connect to elasticsearch and wait for expected status $desiredStatus..."
+      curl -sS "http://localhost:9200/_cluster/health?wait_for_status=$desiredStatus&timeout=60s&pretty"
+    else
+      echo "Trying to connect to elasticsearch and wait for expected status $desiredStatus for index $index"
+      curl -sS "http://localhost:9200/_cluster/health/$index?wait_for_status=$desiredStatus&timeout=60s&pretty"
+    fi
     if [ $? -eq 0 ]; then
         echo "Connected"
     else
@@ -444,3 +495,19 @@ install_script() {
     echo "Installing $script to $ESSCRIPTS"
     cp $script $ESSCRIPTS
 }
+
+# permissions from the user umask with the executable bit set
+executable_privileges_for_user_from_umask() {
+    local user=$1
+    shift
+
+    echo $((0777 & ~$(sudo -E -u $user sh -c umask) | 0111))
+}
+
+# permissions from the user umask without the executable bit set
+file_privileges_for_user_from_umask() {
+    local user=$1
+    shift
+
+    echo $((0777 & ~$(sudo -E -u $user sh -c umask) & ~0111))
+}
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash b/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash
index 8a32c982af1..925beaade09 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash
@@ -32,7 +32,7 @@ install_plugin() {
 
     assert_file_exist "$path"
 
-    sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" install "file://$path"
+    sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" install -batch "file://$path"
 
     assert_file_exist "$ESPLUGINS/$name"
     assert_file_exist "$ESPLUGINS/$name/plugin-descriptor.properties"
@@ -91,16 +91,13 @@ install_jvm_example() {
     #just make sure that everything is the same as $CONFIG_DIR, which was properly set up during install
     config_user=$(find "$ESCONFIG" -maxdepth 0 -printf "%u")
     config_owner=$(find "$ESCONFIG" -maxdepth 0 -printf "%g")
-    config_privileges=$(find "$ESCONFIG" -maxdepth 0 -printf "%m")
-    assert_file "$ESCONFIG/jvm-example" d $config_user $config_owner $config_privileges
-    #the original file has no execute permissions and that must not change, but all other permissions
-    #need to be inherited from the parent config dir. We check this by applying the 111 mask to the config dir privileges.
-    for i in `seq 0 2`; do
-        current_perm_dir=${config_privileges:$i:1}
-        final_perm=$(($current_perm_dir & ~1))
-        expected_file_privileges+=$final_perm
-    done
-    assert_file "$ESCONFIG/jvm-example/example.yaml" f $config_user $config_owner $expected_file_privileges
+    # directories should user the user file-creation mask
+    config_privileges=$(executable_privileges_for_user_from_umask $ESPLUGIN_COMMAND_USER)
+    assert_file "$ESCONFIG/jvm-example" d $config_user $config_owner $(printf "%o" $config_privileges)
+    # config files should not be executable and otherwise use the user
+    # file-creation mask
+    expected_file_privileges=$(file_privileges_for_user_from_umask $ESPLUGIN_COMMAND_USER)
+    assert_file "$ESCONFIG/jvm-example/example.yaml" f $config_user $config_owner $(printf "%o" $expected_file_privileges)
 
     echo "Running jvm-example's bin script...."
     "$ESHOME/bin/jvm-example/test" | grep test
@@ -136,11 +133,14 @@ install_and_check_plugin() {
     fi
 
     install_jvm_plugin $fullName "$(readlink -m $fullName-*.zip)"
+
+    assert_module_or_plugin_directory "$ESPLUGINS/$fullName"
+
     if [ $prefix == 'analysis' ]; then
-        assert_file_exist "$(readlink -m $ESPLUGINS/$fullName/lucene-analyzers-$name-*.jar)"
+        assert_module_or_plugin_file "$ESPLUGINS/$fullName/lucene-analyzers-$name-*.jar"
     fi
     for file in "$@"; do
-        assert_file_exist "$(readlink -m $ESPLUGINS/$fullName/$file)"
+        assert_module_or_plugin_file "$ESPLUGINS/$fullName/$file"
     done
 }
 
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/tar.bash b/qa/vagrant/src/test/resources/packaging/scripts/tar.bash
index f9bcc10525b..56b162cdefe 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/tar.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/tar.bash
@@ -68,6 +68,7 @@ move_elasticsearch() {
 
 # Export some useful paths.
 export_elasticsearch_paths() {
+    export ESMODULES="$ESHOME/modules"
     export ESPLUGINS="$ESHOME/plugins"
     export ESCONFIG="$ESHOME/config"
     export ESSCRIPTS="$ESCONFIG/scripts"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json
index b54bee38c24..68468dd542a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json
@@ -8,6 +8,7 @@
       "parts": {
         "repository": {
           "type" : "list",
+          "required": true,
           "description": "Name of repository from which to fetch the snapshot information"
         }
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
index 587fe027813..7c9cf627530 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
@@ -16,6 +16,10 @@
           "type": "time",
           "description": "Specify timeout for connection to master"
         },
+        "preserve_existing": {
+          "type": "boolean",
+          "description": "Whether to update existing settings. If set to `true` existing settings on an index remain unchanged, the default is `false`"
+        },
         "ignore_unavailable": {
           "type": "boolean",
           "description": "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json
index f09efef7c91..bfe12c981dc 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json
@@ -22,9 +22,14 @@
           "description" : "Explicit write consistency setting for the operation"
         },
         "wait_for_completion": {
-           "type" : "boolean",
-           "default": false,
-           "description" : "Should the request should block until the reindex is complete."
+          "type" : "boolean",
+          "default": false,
+          "description" : "Should the request should block until the reindex is complete."
+        },
+        "requests_per_second": {
+          "type": "float",
+          "default": 0,
+          "description": "The throttle for this request in sub-requests per second. 0 means set no throttle."
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
index 7e8683b3475..5cdeed1b142 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
@@ -31,6 +31,10 @@
         "parent_task": {
           "type" : "number",
           "description" : "Return tasks with specified parent task id. Set to -1 to return all."
+        },
+        "wait_for_completion": {
+          "type": "boolean",
+          "description": "Wait for the matching tasks to complete (default: false)"
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update-by-query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update-by-query.json
index 9d5183ee4f3..fe7fdf8a840 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/update-by-query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update-by-query.json
@@ -105,6 +105,10 @@
           "options" : ["query_then_fetch", "dfs_query_then_fetch"],
           "description" : "Search operation type"
         },
+        "search_timeout": {
+          "type" : "time",
+          "description" : "Explicit timeout for each search request. Defaults to no timeout."
+        },
         "size": {
           "type" : "number",
           "description" : "Number of hits to return (default: 10)"
@@ -194,6 +198,11 @@
            "type" : "boolean",
            "default": false,
            "description" : "Should the request should block until the reindex is complete."
+        },
+        "requests_per_second": {
+          "type": "float",
+          "default": 0,
+          "description": "The throttle for this request in sub-requests per second. 0 means set no throttle."
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml
index b4b5669a8a9..770c5c8b441 100755
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml
@@ -49,56 +49,6 @@
                     -                   \s+
                 $/
 
----
-"Simple alias with yaml body through Accept header":
-
-  - do:
-        indices.create:
-            index: test
-
-  - do:
-        indices.put_alias:
-            index: test
-            name:  test_alias
-
-  - do:
-      cat.aliases: {}
-      headers:
-        Accept: application/yaml
-
-  - match:
-      $body: |
-        /^---\n
-          -\s+alias:\s+"test_alias"\s+
-              index:\s+"test"\s+
-              filter:\s+"-"\s+
-              routing.index:\s+"-"\s+
-              routing.search:\s+"-"\s+$/
----
-"Simple alias with yaml body through format argument":
-
-  - do:
-        indices.create:
-            index: test
-
-  - do:
-        indices.put_alias:
-            index: test
-            name:  test_alias
-
-  - do:
-      cat.aliases:
-        format: yaml
-
-  - match:
-      $body: |
-        /^---\n
-          -\s+alias:\s+"test_alias"\s+
-              index:\s+"test"\s+
-              filter:\s+"-"\s+
-              routing.index:\s+"-"\s+
-              routing.search:\s+"-"\s+$/
-
 ---
 "Complex alias":
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/20_headers.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/20_headers.yaml
new file mode 100644
index 00000000000..b814856144b
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/20_headers.yaml
@@ -0,0 +1,28 @@
+---
+"Simple alias with yaml body through Accept header":
+  - skip:
+      features: headers
+
+  - do:
+        indices.create:
+            index: test
+
+  - do:
+        indices.put_alias:
+            index: test
+            name:  test_alias
+
+  - do:
+      cat.aliases: {}
+      headers:
+        Accept: application/yaml
+
+  - match:
+      $body: |
+        /^---\n
+          -\s+alias:\s+"test_alias"\s+
+              index:\s+"test"\s+
+              filter:\s+"-"\s+
+              routing.index:\s+"-"\s+
+              routing.search:\s+"-"\s+$/
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml
new file mode 100644
index 00000000000..c892891f08f
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml
@@ -0,0 +1,29 @@
+---
+"Simple alias with yaml body through format argument":
+
+  - skip:
+      features: yaml
+
+  - do:
+        indices.create:
+            index: test
+
+  - do:
+        indices.put_alias:
+            index: test
+            name:  test_alias
+
+  - do:
+      cat.aliases:
+        format: yaml
+
+  - match:
+      $body: |
+        /^---\n
+          -\s+alias:\s+"test_alias"\s+
+              index:\s+"test"\s+
+              filter:\s+"-"\s+
+              routing.index:\s+"-"\s+
+              routing.search:\s+"-"\s+$/
+
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yaml
index 432b0e50ae4..effc4c20313 100755
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yaml
@@ -27,7 +27,7 @@
               (
                 index1      \s+
                 \d          \s+                                 # shard
-                \d+ms       \s+                                 # time
+                (?:\d+ms|\d+(?:\.\d+)?s)    \s+                 # time in ms or seconds
                 (store|replica|snapshot|relocating)     \s+     # type
                 (init|index|verify_index|translog|finalize|done) \s+ # stage
                 [-\w./]+    \s+                                 # source_host
@@ -35,14 +35,16 @@
                 [-\w./]+    \s+                                 # repository
                 [-\w./]+    \s+                                 # snapshot
                 \d+         \s+                                 # files
+                \d+         \s+                                 # files_recovered
                 \d+\.\d+%   \s+                                 # files_percent
+                \d+         \s+                                 # files_total
                 \d+         \s+                                 # bytes
+                \d+         \s+                                 # bytes_recovered
                 \d+\.\d+%   \s+                                 # bytes_percent
-                \d+         \s+                                 # total_files
-                \d+         \s+                                 # total_bytes
-                \d+         \s+                                 # translog
-                -?\d+\.\d+% \s+                                 # translog_percent
-                -?\d+                                           # total_translog
+                \d+         \s+                                 # bytes_total
+                -?\d+       \s+                                 # translog_ops
+                \d+         \s+                                 # translog_ops_recovered
+                -?\d+\.\d+%                                     # translog_ops_percent
                 \n
               )+
               $/
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml
index f264928c21b..dfafd833509 100755
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml
@@ -48,11 +48,7 @@
                     merges.total_docs                .+   \n
                     merges.total_size                .+   \n
                     merges.total_time                .+   \n
-                    percolate.current                .+   \n
-                    percolate.memory_size            .+   \n
                     percolate.queries                .+   \n
-                    percolate.time                   .+   \n
-                    percolate.total                  .+   \n
                     refresh.total                    .+   \n
                     refresh.time                     .+   \n
                     search.fetch_current             .+   \n
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yaml
index 8d59e7c139c..233e21bf470 100755
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yaml
@@ -29,14 +29,14 @@
 
   - do:
       cat.thread_pool:
-          h: id,ba,fa,gea,ga,ia,maa,ma,fma,pa
+          h: id,ba,fa,gea,ga,ia,maa,ma,fma
           v: true
           full_id: true
 
   - match:
       $body: |
-               /^  id   \s+  ba   \s+  fa   \s+  gea  \s+  ga   \s+  ia   \s+  maa  \s+  fma   \s+  pa    \n
-                  (\S+  \s+  \d+  \s+  \d+  \s+  \d+  \s+  \d+  \s+  \d+  \s+  \d+  \s+  \d+  \s+  \d+   \n)+  $/
+               /^  id   \s+  ba   \s+  fa   \s+  gea  \s+  ga   \s+  ia   \s+  maa  \s+  fma    \n
+                  (\S+  \s+  \d+  \s+  \d+  \s+  \d+  \s+  \d+  \s+  \d+  \s+  \d+  \s+  \d+   \n)+  $/
 
   - do:
       cat.thread_pool:
@@ -108,16 +108,6 @@
                /^  id  \s+  force_merge.type            \s+  force_merge.active  \s+  force_merge.size  \s+  force_merge.queue  \s+  force_merge.queueSize  \s+  force_merge.rejected  \s+  force_merge.largest  \s+  force_merge.completed  \s+  force_merge.min  \s+  force_merge.max  \s+  force_merge.keepAlive    \n
                   (\S+ \s+  (cached|fixed|scaling)?  \s+              \d+  \s+            \d+  \s+             \d+  \s+                 \d*  \s+                \d+  \s+               \d+  \s+                 \d+  \s+           \d*  \s+           \d*  \s+                 \S*    \n)+  $/
 
-  - do:
-      cat.thread_pool:
-          h: id,percolate.type,percolate.active,percolate.size,percolate.queue,percolate.queueSize,percolate.rejected,percolate.largest,percolate.completed,percolate.min,percolate.max,percolate.keepAlive
-          v: true
-
-  - match:
-      $body: |
-               /^  id  \s+  percolate.type           \s+  percolate.active  \s+  percolate.size  \s+  percolate.queue  \s+  percolate.queueSize  \s+  percolate.rejected  \s+  percolate.largest  \s+  percolate.completed  \s+  percolate.min  \s+  percolate.max  \s+  percolate.keepAlive    \n
-                  (\S+ \s+  (cached|fixed|scaling)?  \s+               \d+  \s+             \d+  \s+              \d+  \s+                  \d*  \s+                 \d+  \s+                \d+  \s+                  \d+  \s+            \d*  \s+            \d*  \s+                  \S*    \n)+  $/
-
   - do:
       cat.thread_pool:
           h: id,refresh.type,refresh.active,refresh.size,refresh.queue,refresh.queueSize,refresh.rejected,refresh.largest,refresh.completed,refresh.min,refresh.max,refresh.keepAlive
@@ -148,16 +138,6 @@
                /^  id  \s+  snapshot.type            \s+  snapshot.active  \s+  snapshot.size  \s+  snapshot.queue  \s+  snapshot.queueSize  \s+  snapshot.rejected  \s+  snapshot.largest  \s+  snapshot.completed  \s+  snapshot.min  \s+  snapshot.max  \s+  snapshot.keepAlive    \n
                   (\S+ \s+  (cached|fixed|scaling)?  \s+              \d+  \s+            \d+  \s+             \d+  \s+                 \d*  \s+                \d+  \s+               \d+  \s+                 \d+  \s+           \d*  \s+           \d*  \s+                 \S*    \n)+  $/
 
-  - do:
-      cat.thread_pool:
-          h: id,suggest.type,suggest.active,suggest.size,suggest.queue,suggest.queueSize,suggest.rejected,suggest.largest,suggest.completed,suggest.min,suggest.max,suggest.keepAlive
-          v: true
-
-  - match:
-      $body: |
-               /^  id  \s+  suggest.type             \s+  suggest.active  \s+  suggest.size  \s+  suggest.queue  \s+  suggest.queueSize  \s+  suggest.rejected  \s+  suggest.largest  \s+  suggest.completed  \s+  suggest.min  \s+  suggest.max  \s+  suggest.keepAlive    \n
-                  (\S+ \s+  (cached|fixed|scaling)?  \s+             \d+  \s+           \d+  \s+            \d+  \s+                \d*  \s+               \d+  \s+              \d+  \s+                \d+  \s+          \d*  \s+          \d*  \s+                \S*    \n)+  $/
-
   - do:
       cat.thread_pool:
           h: id,warmer.type,warmer.active,warmer.size,warmer.queue,warmer.queueSize,warmer.rejected,warmer.largest,warmer.completed,warmer.min,warmer.max,warmer.keepAlive
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_settings/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_settings/10_basic.yaml
index f9a453c90f1..cc9fb95b784 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_settings/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_settings/10_basic.yaml
@@ -54,3 +54,70 @@ setup:
         body:
           number_of_replicas: 1
 
+---
+"Test preserve_existing settings":
+  - do:
+      indices.put_settings:
+        index: test-index
+        body:
+          number_of_replicas: 0
+
+  - do:
+      indices.get_settings:
+        flat_settings: false
+
+  - match:
+      test-index.settings.index.number_of_replicas: "0"
+
+  - do:
+      indices.put_settings:
+        preserve_existing: true
+        index: test-index
+        body:
+          index.number_of_replicas: 1
+          index.translog.durability: "async"
+
+  - do:
+      indices.get_settings:
+        flat_settings: false
+
+
+  - match:
+      test-index.settings.index.number_of_replicas: "0"
+  - match:
+      test-index.settings.index.translog.durability: "async"
+
+  - do:
+      indices.close:
+        index: test-index
+
+  - do:
+      indices.put_settings:
+        preserve_existing: true
+        index: test-index
+        body:
+          index.translog.durability: "request"
+          index.query_string.lenient: "true"
+
+  - do:
+      indices.get_settings:
+        index: test-index
+        flat_settings: false
+
+  - match:
+      test-index.settings.index.query_string.lenient: "true"
+  - match:
+      test-index.settings.index.translog.durability: "async"
+
+  - do:
+       indices.open:
+         index: test-index
+  - do:
+      indices.get_settings:
+        index: test-index
+        flat_settings: false
+
+  - match:
+     test-index.settings.index.query_string.lenient: "true"
+  - match:
+     test-index.settings.index.translog.durability: "async"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yaml
index ed676924f99..7b88ac57080 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yaml
@@ -31,11 +31,9 @@ setup:
   - is_true:  _all.total.warmer
   - is_true:  _all.total.query_cache
   - is_true:  _all.total.fielddata
-  - is_true:  _all.total.percolate
   - is_true:  _all.total.completion
   - is_true:  _all.total.segments
   - is_true:  _all.total.translog
-  - is_true:  _all.total.suggest
   - is_true:  _all.total.recovery
 
 ---
@@ -54,11 +52,9 @@ setup:
   - is_true:  _all.total.warmer
   - is_true:  _all.total.query_cache
   - is_true:  _all.total.fielddata
-  - is_true:  _all.total.percolate
   - is_true:  _all.total.completion
   - is_true:  _all.total.segments
   - is_true:  _all.total.translog
-  - is_true:  _all.total.suggest
   - is_true:  _all.total.recovery
 
 ---
@@ -77,11 +73,9 @@ setup:
   - is_false:  _all.total.warmer
   - is_false:  _all.total.query_cache
   - is_false:  _all.total.fielddata
-  - is_false:  _all.total.percolate
   - is_false:  _all.total.completion
   - is_false:  _all.total.segments
   - is_false:  _all.total.translog
-  - is_false:  _all.total.suggest
   - is_false:  _all.total.recovery
 
 ---
@@ -100,11 +94,9 @@ setup:
   - is_false:  _all.total.warmer
   - is_false:  _all.total.query_cache
   - is_false:  _all.total.fielddata
-  - is_false:  _all.total.percolate
   - is_false:  _all.total.completion
   - is_false:  _all.total.segments
   - is_false:  _all.total.translog
-  - is_false:  _all.total.suggest
   - is_false:  _all.total.recovery
 
 
@@ -124,10 +116,8 @@ setup:
   - is_false:  _all.total.warmer
   - is_false:  _all.total.query_cache
   - is_false:  _all.total.fielddata
-  - is_false:  _all.total.percolate
   - is_false:  _all.total.completion
   - is_false:  _all.total.segments
   - is_false:  _all.total.translog
-  - is_false:  _all.total.suggest
   - is_true:   _all.total.recovery
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/10_crud.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/10_crud.yaml
index ced2e9e4850..b494161aff1 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/10_crud.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/10_crud.yaml
@@ -1,28 +1,3 @@
----
-"Check availability of default processors":
-  - do:
-      cluster.state: {}
-
-  - set: {master_node: master}
-
-  - do:
-      nodes.info: {}
-
-  - match:  { nodes.$master.ingest.processors.0.type: date }
-  - match:  { nodes.$master.ingest.processors.1.type: uppercase }
-  - match:  { nodes.$master.ingest.processors.2.type: set }
-  - match:  { nodes.$master.ingest.processors.3.type: lowercase }
-  - match:  { nodes.$master.ingest.processors.4.type: gsub }
-  - match:  { nodes.$master.ingest.processors.5.type: convert }
-  - match:  { nodes.$master.ingest.processors.6.type: remove }
-  - match:  { nodes.$master.ingest.processors.7.type: fail }
-  - match:  { nodes.$master.ingest.processors.8.type: foreach }
-  - match:  { nodes.$master.ingest.processors.9.type: split }
-  - match:  { nodes.$master.ingest.processors.10.type: trim }
-  - match:  { nodes.$master.ingest.processors.11.type: rename }
-  - match:  { nodes.$master.ingest.processors.12.type: join }
-  - match:  { nodes.$master.ingest.processors.13.type: append }
-
 ---
 "Test basic pipeline crud":
   - do:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/70_bulk.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/70_bulk.yaml
index b363f018667..d834d86b2ea 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/70_bulk.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/70_bulk.yaml
@@ -68,6 +68,24 @@ setup:
   - is_false: _source.field1
   - is_false: _source.field2
 
+  - do:
+      cluster.state: {}
+    # Get master node id
+  - set: { master_node: master }
+
+  - do:
+      nodes.stats:
+        metric: [ ingest ]
+  #we can't assert anything here since we might have more than one node in the cluster
+  - gte: {nodes.$master.ingest.total.count: 0}
+  - gte: {nodes.$master.ingest.total.failed: 0}
+  - gte: {nodes.$master.ingest.total.time_in_millis: 0}
+  - match: {nodes.$master.ingest.total.current: 0}
+  - gte: {nodes.$master.ingest.pipelines.pipeline1.count: 0}
+  - match: {nodes.$master.ingest.pipelines.pipeline1.failed: 0}
+  - gte: {nodes.$master.ingest.pipelines.pipeline1.time_in_millis: 0}
+  - match: {nodes.$master.ingest.pipelines.pipeline1.current: 0}
+
 ---
 "Test bulk request with default pipeline":
 
@@ -88,6 +106,24 @@ setup:
           - f1: v2
   - gte: { ingest_took: 0 }
 
+  - do:
+      cluster.state: {}
+  # Get master node id
+  - set: { master_node: master }
+
+  - do:
+      nodes.stats:
+        metric: [ ingest ]
+  #we can't assert anything here since we might have more than one node in the cluster
+  - gte: {nodes.$master.ingest.total.count: 0}
+  - gte: {nodes.$master.ingest.total.failed: 0}
+  - gte: {nodes.$master.ingest.total.time_in_millis: 0}
+  - match: {nodes.$master.ingest.total.current: 0}
+  - gte: {nodes.$master.ingest.pipelines.pipeline2.count: 0}
+  - match: {nodes.$master.ingest.pipelines.pipeline2.failed: 0}
+  - gte: {nodes.$master.ingest.pipelines.pipeline2.time_in_millis: 0}
+  - match: {nodes.$master.ingest.pipelines.pipeline2.current: 0}
+
   - do:
       get:
         index: test_index
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/percolate/16_existing_doc.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/percolate/16_existing_doc.yaml
index 2430fe63a02..c6f12131f35 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/percolate/16_existing_doc.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/percolate/16_existing_doc.yaml
@@ -108,5 +108,5 @@
             term:
               tag: tag1
 
-  - match: {'matches': [{_index: percolator_index, _id: test_percolator}]}
+  - match: {'matches': [{_index: percolator_index, _id: test_percolator, _score: 1.0}]}
 
diff --git a/settings.gradle b/settings.gradle
index f2518e69b12..d03cac653ee 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -11,6 +11,7 @@ List projects = [
   'test:framework',
   'test:fixtures:example-fixture',
   'test:fixtures:hdfs-fixture',
+  'test:logger-usage',
   'modules:ingest-grok',
   'modules:lang-expression',
   'modules:lang-groovy',
@@ -38,6 +39,7 @@ List projects = [
   'plugins:repository-s3',
   'plugins:jvm-example',
   'plugins:store-smb',
+  'qa:backwards-5.0',
   'qa:evil-tests',
   'qa:smoke-test-client',
   'qa:smoke-test-multinode',
diff --git a/test/build.gradle b/test/build.gradle
index 564f8673307..7feb332b717 100644
--- a/test/build.gradle
+++ b/test/build.gradle
@@ -27,11 +27,11 @@ subprojects {
   apply plugin: 'elasticsearch.build'
 
   
-  // the main files are actually test files, so use the appopriate forbidden api sigs
+  // the main files are actually test files, so use the appropriate forbidden api sigs
   forbiddenApisMain {
-    bundledSignatures = ['jdk-unsafe', 'jdk-deprecated']
-    signaturesURLs = [PrecommitTasks.getResource('/forbidden/all-signatures.txt'),
-                      PrecommitTasks.getResource('/forbidden/test-signatures.txt')]
+    signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt'),
+                      PrecommitTasks.getResource('/forbidden/es-signatures.txt'),
+                      PrecommitTasks.getResource('/forbidden/es-test-signatures.txt')]
   }
 
   // TODO: should we have licenses for our test deps?
diff --git a/test/framework/build.gradle b/test/framework/build.gradle
index 355459f99f5..af65c9ff7c9 100644
--- a/test/framework/build.gradle
+++ b/test/framework/build.gradle
@@ -36,9 +36,8 @@ dependencies {
 compileJava.options.compilerArgs << '-Xlint:-cast,-rawtypes,-try,-unchecked'
 compileTestJava.options.compilerArgs << '-Xlint:-rawtypes'
 
-// the main files are actually test files, so use the appopriate forbidden api sigs
+// the main files are actually test files, so use the appropriate forbidden api sigs
 forbiddenApisMain {
-  bundledSignatures = ['jdk-unsafe', 'jdk-deprecated']
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/all-signatures.txt'),
                     PrecommitTasks.getResource('/forbidden/test-signatures.txt')]
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
index aa77c670a42..68eb0420b39 100644
--- a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
+++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
@@ -166,7 +166,7 @@ public class BootstrapForTesting {
     }
 
     /**
-     * we dont know which codesources belong to which plugin, so just remove the permission from key codebases
+     * we don't know which codesources belong to which plugin, so just remove the permission from key codebases
      * like core, test-framework, etc. this way tests fail if accesscontroller blocks are missing.
      */
     @SuppressForbidden(reason = "accesses fully qualified URLs to configure security")
diff --git a/test/framework/src/main/java/org/elasticsearch/cli/CommandTestCase.java b/test/framework/src/main/java/org/elasticsearch/cli/CommandTestCase.java
new file mode 100644
index 00000000000..e9c6a2eec9c
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/cli/CommandTestCase.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cli;
+
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+/**
+ * A base test case for cli tools.
+ */
+public abstract class CommandTestCase extends ESTestCase {
+
+    /** The terminal that execute uses. */
+    protected final MockTerminal terminal = new MockTerminal();
+
+    /** The last command that was executed. */
+    protected Command command;
+
+    @Before
+    public void resetTerminal() {
+        terminal.reset();
+        terminal.setVerbosity(Terminal.Verbosity.NORMAL);
+    }
+
+    /** Creates a Command to test execution. */
+    protected abstract Command newCommand();
+
+    /**
+     * Runs the command with the given args.
+     *
+     * Output can be found in {@link #terminal}.
+     * The command created can be found in {@link #command}.
+     */
+    public String execute(String... args) throws Exception {
+        command = newCommand();
+        command.mainWithoutErrorHandling(args, terminal);
+        return terminal.getOutput();
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/common/cli/MockTerminal.java b/test/framework/src/main/java/org/elasticsearch/cli/MockTerminal.java
similarity index 87%
rename from test/framework/src/main/java/org/elasticsearch/common/cli/MockTerminal.java
rename to test/framework/src/main/java/org/elasticsearch/cli/MockTerminal.java
index 3b2903b3fab..bd8bd493cea 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/cli/MockTerminal.java
+++ b/test/framework/src/main/java/org/elasticsearch/cli/MockTerminal.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.common.cli;
+package org.elasticsearch.cli;
 
 import java.io.ByteArrayOutputStream;
 import java.io.OutputStreamWriter;
@@ -45,7 +45,7 @@ public class MockTerminal extends Terminal {
     @Override
     public String readText(String prompt) {
         if (textInput.isEmpty()) {
-            return null;
+            throw new IllegalStateException("No text input configured for prompt [" + prompt + "]");
         }
         return textInput.removeFirst();
     }
@@ -53,7 +53,7 @@ public class MockTerminal extends Terminal {
     @Override
     public char[] readSecret(String prompt) {
         if (secretInput.isEmpty()) {
-            return null;
+            throw new IllegalStateException("No secret input configured for prompt [" + prompt + "]");
         }
         return secretInput.removeFirst().toCharArray();
     }
@@ -78,8 +78,10 @@ public class MockTerminal extends Terminal {
         return buffer.toString("UTF-8");
     }
 
-    /** Wipes the output. */
-    public void resetOutput() {
+    /** Wipes the input and output. */
+    public void reset() {
         buffer.reset();
+        textInput.clear();
+        secretInput.clear();
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
index 3e9b0c09cb2..5b973f67abc 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
@@ -27,6 +27,7 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -74,7 +75,7 @@ public class MockInternalClusterInfoService extends InternalClusterInfoService {
             null, null, null, null, null,
             fsInfo,
             null, null, null,
-            null, null);
+            null, null, null);
     }
 
     @Inject
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java
index b7984416fae..d15773db545 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java
@@ -30,7 +30,7 @@ import org.elasticsearch.test.ESTestCase;
 public class TestShardRouting {
 
     public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, boolean primary, ShardRoutingState state) {
-        return newShardRouting(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), shardId, currentNodeId,primary, state);
+        return newShardRouting(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), shardId, currentNodeId, primary, state);
     }
 
     public static ShardRouting newShardRouting(Index index, int shardId, String currentNodeId, boolean primary, ShardRoutingState state) {
diff --git a/test/framework/src/main/java/org/elasticsearch/common/cli/CliToolTestCase.java b/test/framework/src/main/java/org/elasticsearch/common/cli/CliToolTestCase.java
deleted file mode 100644
index 06c197a05da..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/common/cli/CliToolTestCase.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.cli;
-
-import java.io.IOException;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.StreamsUtils;
-import org.junit.After;
-import org.junit.Before;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.isEmptyString;
-import static org.hamcrest.Matchers.not;
-
-public abstract class CliToolTestCase extends ESTestCase {
-
-    @Before
-    @SuppressForbidden(reason = "sets es.default.path.home during tests")
-    public void setPathHome() {
-        System.setProperty("es.default.path.home", createTempDir().toString());
-    }
-
-    @After
-    @SuppressForbidden(reason = "clears es.default.path.home during tests")
-    public void clearPathHome() {
-        System.clearProperty("es.default.path.home");
-    }
-
-    public static String[] args(String command) {
-        if (!Strings.hasLength(command)) {
-            return Strings.EMPTY_ARRAY;
-        }
-        return command.split("\\s+");
-    }
-
-    public static void assertTerminalOutputContainsHelpFile(MockTerminal terminal, String classPath) throws IOException {
-        String output = terminal.getOutput();
-        assertThat(output, not(isEmptyString()));
-        String expectedDocs = StreamsUtils.copyToStringFromClasspath(classPath);
-        // convert to *nix newlines as MockTerminal used for tests also uses *nix newlines
-        expectedDocs = expectedDocs.replace("\r\n", "\n");
-        assertThat(output, containsString(expectedDocs));
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java
new file mode 100644
index 00000000000..8a28a16220c
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+
+// this sucks how much must be overridden just do get a dummy field mapper...
+public class MockFieldMapper extends FieldMapper {
+    static Settings dummySettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
+
+    public MockFieldMapper(String fullName) {
+        this(fullName, new FakeFieldType());
+    }
+
+    public MockFieldMapper(String fullName, MappedFieldType fieldType) {
+        super(findSimpleName(fullName), setName(fullName, fieldType), setName(fullName, fieldType), dummySettings,
+            MultiFields.empty(), new CopyTo.Builder().build());
+    }
+
+    static MappedFieldType setName(String fullName, MappedFieldType fieldType) {
+        fieldType.setName(fullName);
+        return fieldType;
+    }
+
+    static String findSimpleName(String fullName) {
+        int ndx = fullName.lastIndexOf('.');
+        return fullName.substring(ndx + 1);
+    }
+
+    static class FakeFieldType extends MappedFieldType {
+        public FakeFieldType() {
+        }
+
+        protected FakeFieldType(FakeFieldType ref) {
+            super(ref);
+        }
+
+        @Override
+        public MappedFieldType clone() {
+            return new FakeFieldType(this);
+        }
+
+        @Override
+        public String typeName() {
+            return "faketype";
+        }
+    }
+
+    @Override
+    protected String contentType() {
+        return null;
+    }
+
+    @Override
+    protected void parseCreateField(ParseContext context, List list) throws IOException {
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
index 7496bfb8263..4022366c1d0 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.search;
 
 import org.elasticsearch.cache.recycler.PageCacheRecycler;
-import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -33,6 +33,7 @@ import org.elasticsearch.search.dfs.DfsPhase;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.query.QueryPhase;
+import org.elasticsearch.search.suggest.Suggesters;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.HashMap;
@@ -69,9 +70,9 @@ public class MockSearchService extends SearchService {
     public MockSearchService(Settings settings, ClusterSettings clusterSettings, ClusterService clusterService,
             IndicesService indicesService, ThreadPool threadPool, ScriptService scriptService, PageCacheRecycler pageCacheRecycler,
             BigArrays bigArrays, DfsPhase dfsPhase, QueryPhase queryPhase, FetchPhase fetchPhase,
-            AggregatorParsers aggParsers) {
+            AggregatorParsers aggParsers, Suggesters suggesters) {
         super(settings, clusterSettings, clusterService, indicesService, threadPool, scriptService, pageCacheRecycler, bigArrays, dfsPhase,
-                queryPhase, fetchPhase, aggParsers);
+                queryPhase, fetchPhase, aggParsers, suggesters);
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java
index 15b72c4cccd..916adc142c8 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java
@@ -88,11 +88,11 @@ public final class CorruptionUtils {
             // we need to add assumptions here that the checksums actually really don't match there is a small chance to get collisions
             // in the checksum which is ok though....
             StringBuilder msg = new StringBuilder();
-            msg.append("Checksum before: [").append(checksumBeforeCorruption).append("]");
-            msg.append(" after: [").append(checksumAfterCorruption).append("]");
-            msg.append(" checksum value after corruption: ").append(actualChecksumAfterCorruption).append("]");
-            msg.append(" file: ").append(fileToCorrupt.getFileName()).append(" length: ").append(dir.fileLength(fileToCorrupt.getFileName().toString()));
-            logger.info(msg.toString());
+            msg.append("before: [").append(checksumBeforeCorruption).append("] ");
+            msg.append("after: [").append(checksumAfterCorruption).append("] ");
+            msg.append("checksum value after corruption: ").append(actualChecksumAfterCorruption).append("] ");
+            msg.append("file: ").append(fileToCorrupt.getFileName()).append(" length: ").append(dir.fileLength(fileToCorrupt.getFileName().toString()));
+            logger.info("Checksum {}", msg);
             assumeTrue("Checksum collision - " + msg.toString(),
                     checksumAfterCorruption != checksumBeforeCorruption // collision
                             || actualChecksumAfterCorruption != checksumBeforeCorruption); // checksum corrupted
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java
index f653819c140..1a38e32cf1a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java
@@ -33,7 +33,8 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation;
-import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocators;
+import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
+import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
@@ -79,19 +80,19 @@ public abstract class ESAllocationTestCase extends ESTestCase {
     public static MockAllocationService createAllocationService(Settings settings, ClusterSettings clusterSettings, Random random) {
         return new MockAllocationService(settings,
                 randomAllocationDeciders(settings, clusterSettings, random),
-                new ShardsAllocators(settings, NoopGatewayAllocator.INSTANCE), EmptyClusterInfoService.INSTANCE);
+                NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings), EmptyClusterInfoService.INSTANCE);
     }
 
     public static MockAllocationService createAllocationService(Settings settings, ClusterInfoService clusterInfoService) {
         return new MockAllocationService(settings,
                 randomAllocationDeciders(settings, new ClusterSettings(Settings.Builder.EMPTY_SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), getRandom()),
-                new ShardsAllocators(settings, NoopGatewayAllocator.INSTANCE), clusterInfoService);
+                NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings), clusterInfoService);
     }
 
-    public static MockAllocationService createAllocationService(Settings settings, GatewayAllocator allocator) {
+    public static MockAllocationService createAllocationService(Settings settings, GatewayAllocator gatewayAllocator) {
         return new MockAllocationService(settings,
                 randomAllocationDeciders(settings, new ClusterSettings(Settings.Builder.EMPTY_SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), getRandom()),
-                new ShardsAllocators(settings, allocator), EmptyClusterInfoService.INSTANCE);
+                gatewayAllocator, new BalancedShardsAllocator(settings), EmptyClusterInfoService.INSTANCE);
     }
 
 
@@ -193,8 +194,9 @@ public abstract class ESAllocationTestCase extends ESTestCase {
 
         private Long nanoTimeOverride = null;
 
-        public MockAllocationService(Settings settings, AllocationDeciders allocationDeciders, ShardsAllocators shardsAllocators, ClusterInfoService clusterInfoService) {
-            super(settings, allocationDeciders, shardsAllocators, clusterInfoService);
+        public MockAllocationService(Settings settings, AllocationDeciders allocationDeciders, GatewayAllocator gatewayAllocator,
+                                     ShardsAllocator shardsAllocator, ClusterInfoService clusterInfoService) {
+            super(settings, allocationDeciders, gatewayAllocator, shardsAllocator, clusterInfoService);
         }
 
         public void setNanoTimeOverride(long nanoTime) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index a3161f4090f..6920ba4a8e0 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -41,6 +41,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse;
@@ -56,7 +57,6 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.AdminClient;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Requests;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -68,6 +68,7 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
@@ -77,6 +78,7 @@ import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
@@ -95,20 +97,21 @@ import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
 import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.MockEngineFactoryPlugin;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.codec.CodecService;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.MappedFieldType.Loading;
-import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
+import org.elasticsearch.index.IndexWarmer;
 import org.elasticsearch.index.MergePolicyConfig;
 import org.elasticsearch.index.MergeSchedulerConfig;
+import org.elasticsearch.index.MockEngineFactoryPlugin;
+import org.elasticsearch.index.codec.CodecService;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.index.IndexWarmer;
 import org.elasticsearch.indices.IndicesRequestCache;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.store.IndicesStore;
 import org.elasticsearch.node.NodeMocksPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -196,7 +199,7 @@ import static org.hamcrest.Matchers.startsWith;
  * should be used, here is an example:
  * <pre>
  *
- * {@literal @}ClusterScope(scope=Scope.TEST) public class SomeIT extends ESIntegTestCase {
+ * {@literal @}NodeScope(scope=Scope.TEST) public class SomeIT extends ESIntegTestCase {
  * public void testMethod() {}
  * }
  * </pre>
@@ -207,7 +210,7 @@ import static org.hamcrest.Matchers.startsWith;
  * determined at random and can change across tests. The {@link ClusterScope} allows configuring the initial number of nodes
  * that are created before the tests start.
  *  <pre>
- * {@literal @}ClusterScope(scope=Scope.SUITE, numDataNodes=3)
+ * {@literal @}NodeScope(scope=Scope.SUITE, numDataNodes=3)
  * public class SomeIT extends ESIntegTestCase {
  * public void testMethod() {}
  * }
@@ -268,7 +271,8 @@ public abstract class ESIntegTestCase extends ESTestCase {
      * The value of this seed can be used to initialize a random context for a specific index.
      * It's set once per test via a generic index template.
      */
-    public static final Setting<Long> INDEX_TEST_SEED_SETTING = Setting.longSetting("index.tests.seed", 0, Long.MIN_VALUE, false, Setting.Scope.INDEX);
+    public static final Setting<Long> INDEX_TEST_SEED_SETTING =
+        Setting.longSetting("index.tests.seed", 0, Long.MIN_VALUE, Property.IndexScope);
 
     /**
      * A boolean value to enable or disable mock modules. This is useful to test the
@@ -354,10 +358,6 @@ public abstract class ESIntegTestCase extends ESTestCase {
         }
     }
 
-    private Loading randomLoadingValues() {
-        return randomFrom(Loading.values());
-    }
-
     /**
      * Creates a randomized index template. This template is used to pass in randomized settings on a
      * per index basis. Allows to enable/disable the randomization for number of shards and replicas
@@ -384,6 +384,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
             } else {
                 randomSettingsBuilder.put("index.codec", CodecService.LUCENE_DEFAULT_CODEC);
             }
+
             XContentBuilder mappings = null;
             if (frequently() && randomDynamicTemplates()) {
                 mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_");
@@ -392,65 +393,21 @@ public abstract class ESIntegTestCase extends ESTestCase {
                             .field("enabled", randomBoolean());
                     mappings.endObject();
                 }
-                mappings.startArray("dynamic_templates")
-                        .startObject()
-                        .startObject("template-strings")
-                        .field("match_mapping_type", "string")
-                        .startObject("mapping")
-                        .startObject("fielddata")
-                        .field(Loading.KEY, randomLoadingValues())
-                        .endObject()
-                        .endObject()
-                        .endObject()
-                        .endObject()
-                        .startObject()
-                        .startObject("template-longs")
-                        .field("match_mapping_type", "long")
-                        .startObject("mapping")
-                        .startObject("fielddata")
-                        .field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER))
-                        .endObject()
-                        .endObject()
-                        .endObject()
-                        .endObject()
-                        .startObject()
-                        .startObject("template-doubles")
-                        .field("match_mapping_type", "double")
-                        .startObject("mapping")
-                        .startObject("fielddata")
-                        .field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER))
-                        .endObject()
-                        .endObject()
-                        .endObject()
-                        .endObject()
-                        .startObject()
-                        .startObject("template-geo_points")
-                        .field("match_mapping_type", "geo_point")
-                        .startObject("mapping")
-                        .startObject("fielddata")
-                        .field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER))
-                        .endObject()
-                        .endObject()
-                        .endObject()
-                        .endObject()
-                        .startObject()
-                        .startObject("template-booleans")
-                        .field("match_mapping_type", "boolean")
-                        .startObject("mapping")
-                        .startObject("fielddata")
-                        .field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER))
-                        .endObject()
-                        .endObject()
-                        .endObject()
-                        .endObject()
-                        .endArray();
                 mappings.endObject().endObject();
             }
 
             for (String setting : randomSettingsBuilder.internalMap().keySet()) {
                 assertThat("non index. prefix setting set on index template, its a node setting...", setting, startsWith("index."));
             }
+            // always default delayed allocation to 0 to make sure we have tests are not delayed
+            randomSettingsBuilder.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), 0);
+            if (randomBoolean()) {
+                randomSettingsBuilder.put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), randomBoolean() ? IndexModule.INDEX_QUERY_CACHE : IndexModule.NONE_QUERY_CACHE);
+            }
 
+            if (randomBoolean()) {
+                randomSettingsBuilder.put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), randomBoolean());
+            }
             PutIndexTemplateRequestBuilder putTemplate = client().admin().indices()
                     .preparePutTemplate("random_index_template")
                     .setTemplate("*")
@@ -467,7 +424,6 @@ public abstract class ESIntegTestCase extends ESTestCase {
     protected Settings.Builder setRandomIndexSettings(Random random, Settings.Builder builder) {
         setRandomIndexMergeSettings(random, builder);
         setRandomIndexTranslogSettings(random, builder);
-        setRandomIndexNormsLoading(random, builder);
 
         if (random.nextBoolean()) {
             builder.put(MergeSchedulerConfig.AUTO_THROTTLE_SETTING.getKey(), false);
@@ -506,13 +462,6 @@ public abstract class ESIntegTestCase extends ESTestCase {
         return builder;
     }
 
-    private static Settings.Builder setRandomIndexNormsLoading(Random random, Settings.Builder builder) {
-        if (random.nextBoolean()) {
-            builder.put(IndexWarmer.INDEX_NORMS_LOADING_SETTING.getKey(), RandomPicks.randomFrom(random, Arrays.asList(MappedFieldType.Loading.EAGER, MappedFieldType.Loading.LAZY)));
-        }
-        return builder;
-    }
-
     private static Settings.Builder setRandomIndexTranslogSettings(Random random, Settings.Builder builder) {
         if (random.nextBoolean()) {
             builder.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(RandomInts.randomIntBetween(random, 1, 300), ByteSizeUnit.MB));
@@ -736,6 +685,8 @@ public abstract class ESIntegTestCase extends ESTestCase {
             logger.info("using custom data_path for index: [{}]", dataPath);
             builder.put(IndexMetaData.SETTING_DATA_PATH, dataPath);
         }
+        // always default delayed allocation to 0 to make sure we have tests are not delayed
+        builder.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), 0);
         return builder.build();
     }
 
@@ -836,7 +787,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
         assertThat(nodes, Matchers.not(Matchers.emptyIterable()));
         for (String node : nodes) {
             IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node);
-            IndexService indexService = indicesService.indexService(index);
+            IndexService indexService = indicesService.indexService(resolveIndex(index));
             assertThat("index service doesn't exists on " + node, indexService, notNullValue());
             DocumentMapper documentMapper = indexService.mapperService().documentMapper(type);
             assertThat("document mapper doesn't exists on " + node, documentMapper, notNullValue());
@@ -881,7 +832,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
                 sb.append("\n-> _index: [").append(hit.getIndex()).append("] type [").append(hit.getType())
                         .append("] id [").append(hit.id()).append("]");
             }
-            logger.warn(sb.toString());
+            logger.warn("{}", sb);
             fail(failMsg);
         }
     }
@@ -1096,7 +1047,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
             // remove local node reference
             masterClusterState = ClusterState.Builder.fromBytes(masterClusterStateBytes, null);
             Map<String, Object> masterStateMap = convertToMap(masterClusterState);
-            int masterClusterStateSize = masterClusterState.toString().length();
+            int masterClusterStateSize = ClusterState.Builder.toBytes(masterClusterState).length;
             String masterId = masterClusterState.nodes().masterNodeId();
             for (Client client : cluster().getClients()) {
                 ClusterState localClusterState = client.admin().cluster().prepareState().all().setLocal(true).get().getState();
@@ -1104,7 +1055,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
                 // remove local node reference
                 localClusterState = ClusterState.Builder.fromBytes(localClusterStateBytes, null);
                 final Map<String, Object> localStateMap = convertToMap(localClusterState);
-                final int localClusterStateSize = localClusterState.toString().length();
+                final int localClusterStateSize = ClusterState.Builder.toBytes(localClusterState).length;
                 // Check that the non-master node has the same version of the cluster state as the master and
                 // that the master node matches the master (otherwise there is no requirement for the cluster state to match)
                 if (masterClusterState.version() == localClusterState.version() && masterId.equals(localClusterState.nodes().masterNodeId())) {
@@ -1696,7 +1647,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
 
     /** Helper method to create list of plugins without specifying generic types. */
     @SafeVarargs
-    @SuppressWarnings("varargs") // due to type erasure, the varargs type is non-reifiable, which casues this warning
+    @SuppressWarnings("varargs") // due to type erasure, the varargs type is non-reifiable, which causes this warning
     protected final Collection<Class<? extends Plugin>> pluginList(Class<? extends Plugin>... plugins) {
         return Arrays.asList(plugins);
     }
@@ -1704,7 +1655,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
     /**
      * This method is used to obtain additional settings for clients created by the internal cluster.
      * These settings will be applied on the client in addition to some randomized settings defined in
-     * the cluster. These setttings will also override any other settings the internal cluster might
+     * the cluster. These settings will also override any other settings the internal cluster might
      * add by default.
      */
     protected Settings transportClientSettings() {
@@ -1840,7 +1791,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
         }
         @Override
         public String description() {
-            return "a test plugin that registeres index.tests.seed as an index setting";
+            return "a test plugin that registers index.tests.seed as an index setting";
         }
         public void onModule(SettingsModule module) {
             module.registerSetting(INDEX_TEST_SEED_SETTING);
@@ -1981,7 +1932,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
     @After
     public final void after() throws Exception {
         printTestMessage("finished");
-        // Deleting indices is going to clear search contexts implicitely so we
+        // Deleting indices is going to clear search contexts implicitly so we
         // need to check that there are no more in-flight search contexts before
         // we remove indices
         super.ensureAllSearchContextsReleased();
@@ -2041,7 +1992,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
      * of the provided index.
      */
     protected String routingKeyForShard(String index, String type, int shard) {
-        return internalCluster().routingKeyForShard(index, type, shard, getRandom());
+        return internalCluster().routingKeyForShard(resolveIndex(index), type, shard, getRandom());
     }
 
     /**
@@ -2144,4 +2095,11 @@ public abstract class ESIntegTestCase extends ESTestCase {
     public @interface SuppressNetworkMode {
     }
 
+    public static Index resolveIndex(String index) {
+        GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().setIndices(index).get();
+        assertTrue("index " + index + " not found", getIndexResponse.getSettings().containsKey(index));
+        String uuid = getIndexResponse.getSettings().get(index).get(IndexMetaData.SETTING_INDEX_UUID);
+        return new Index(index, uuid);
+    }
+
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
index fc713400262..73576c9a840 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
@@ -22,6 +22,8 @@ import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder;
 import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Requests;
@@ -31,13 +33,13 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.node.MockNode;
@@ -83,6 +85,12 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
         // SERVICE_UNAVAILABLE/1/state not recovered / initialized block
         ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForGreenStatus().get();
         assertFalse(clusterHealthResponse.isTimedOut());
+        client().admin().indices()
+            .preparePutTemplate("random_index_template")
+            .setTemplate("*")
+            .setOrder(0)
+            .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get();
     }
 
     private static void stopNode() throws IOException {
@@ -108,7 +116,7 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
     public void setUp() throws Exception {
         super.setUp();
         // Create the node lazily, on the first test. This is ok because we do not randomize any settings,
-        // only the cluster name. This allows us to have overriden properties for plugins and the version to use.
+        // only the cluster name. This allows us to have overridden properties for plugins and the version to use.
         if (NODE == null) {
             startNode();
         }
@@ -153,11 +161,16 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
 
     /** Helper method to create list of plugins without specifying generic types. */
     @SafeVarargs
-    @SuppressWarnings("varargs") // due to type erasure, the varargs type is non-reifiable, which casues this warning
+    @SuppressWarnings("varargs") // due to type erasure, the varargs type is non-reifiable, which causes this warning
     protected final Collection<Class<? extends Plugin>> pluginList(Class<? extends Plugin>... plugins) {
         return Arrays.asList(plugins);
     }
 
+    /** Additional settings to add when creating the node. Also allows overriding the default settings. */
+    protected Settings nodeSettings() {
+        return Settings.EMPTY;
+    }
+
     private Node newNode() {
         Settings settings = Settings.builder()
             .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong()))
@@ -166,8 +179,7 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
             // This needs to tie into the ESIntegTestCase#indexSettings() method
             .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir().getParent())
             .put("node.name", nodeName())
-            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+
             .put("script.inline", "true")
             .put("script.indexed", "true")
             .put(EsExecutors.PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created
@@ -175,6 +187,7 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
             .put(Node.NODE_LOCAL_SETTING.getKey(), true)
             .put(Node.NODE_DATA_SETTING.getKey(), true)
             .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING.getKey(), true) // make sure we get what we set :)
+            .put(nodeSettings()) // allow test cases to provide their own settings or override these
             .build();
         Node build = new MockNode(settings, getVersion(), getPlugins());
         build.start();
@@ -255,15 +268,22 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
         assertThat(health.getStatus(), lessThanOrEqualTo(ClusterHealthStatus.YELLOW));
         assertThat("Cluster must be a single node cluster", health.getNumberOfDataNodes(), equalTo(1));
         IndicesService instanceFromNode = getInstanceFromNode(IndicesService.class);
-        return instanceFromNode.indexServiceSafe(index);
+        return instanceFromNode.indexServiceSafe(resolveIndex(index));
+    }
+
+    public Index resolveIndex(String index) {
+        GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().setIndices(index).get();
+        assertTrue("index " + index + " not found", getIndexResponse.getSettings().containsKey(index));
+        String uuid = getIndexResponse.getSettings().get(index).get(IndexMetaData.SETTING_INDEX_UUID);
+        return new Index(index, uuid);
     }
 
     /**
      * Create a new search context.
      */
     protected SearchContext createSearchContext(IndexService indexService) {
-        BigArrays bigArrays = indexService.getIndexServices().getBigArrays();
-        ThreadPool threadPool = indexService.getIndexServices().getThreadPool();
+        BigArrays bigArrays = indexService.getBigArrays();
+        ThreadPool threadPool = indexService.getThreadPool();
         PageCacheRecycler pageCacheRecycler = node().injector().getInstance(PageCacheRecycler.class);
         ScriptService scriptService = node().injector().getInstance(ScriptService.class);
         return new TestSearchContext(threadPool, pageCacheRecycler, bigArrays, scriptService, indexService);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 84d88733802..d2cc23349fc 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -29,7 +29,6 @@ import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
-import junit.framework.AssertionFailedError;
 import org.apache.lucene.uninverting.UninvertingReader;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
@@ -46,10 +45,14 @@ import org.elasticsearch.common.io.PathUtilsForTesting;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.search.MockSearchService;
 import org.elasticsearch.test.junit.listeners.LoggingListener;
 import org.elasticsearch.test.junit.listeners.ReproduceInfoPrinter;
@@ -67,6 +70,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
@@ -75,7 +79,10 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BooleanSupplier;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
 
+import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -130,7 +137,7 @@ public abstract class ESTestCase extends LuceneTestCase {
     protected void afterIfFailed(List<Throwable> errors) {
     }
 
-    /** called after a test is finished, but only if succesfull */
+    /** called after a test is finished, but only if successful */
     protected void afterIfSuccessful() throws Exception {
     }
 
@@ -282,10 +289,10 @@ public abstract class ESTestCase extends LuceneTestCase {
      * Returns a double value in the interval [start, end) if lowerInclusive is
      * set to true, (start, end) otherwise.
      *
-     * @param start lower bound of interval to draw uniformly distributed random numbers from
-     * @param end upper bound
+     * @param start          lower bound of interval to draw uniformly distributed random numbers from
+     * @param end            upper bound
      * @param lowerInclusive whether or not to include lower end of the interval
-     * */
+     */
     public static double randomDoubleBetween(double start, double end, boolean lowerInclusive) {
         double result = 0.0;
 
@@ -396,6 +403,26 @@ public abstract class ESTestCase extends LuceneTestCase {
         return randomTimeValue(1, 1000);
     }
 
+    /**
+     * helper to randomly perform on <code>consumer</code> with <code>value</code>
+     */
+    public static <T> void maybeSet(Consumer<T> consumer, T value) {
+        if (randomBoolean()) {
+            consumer.accept(value);
+        }
+    }
+
+    /**
+     * helper to get a random value in a certain range that's different from the input
+     */
+    public static <T> T randomValueOtherThan(T input, Supplier<T> randomSupplier) {
+        T randomValue = null;
+        do {
+            randomValue = randomSupplier.get();
+        } while (randomValue.equals(input));
+        return randomValue;
+    }
+
     /**
      * Runs the code block for 10 seconds waiting for no assertion to trip.
      */
@@ -555,12 +582,27 @@ public abstract class ESTestCase extends LuceneTestCase {
      * Returns size random values
      */
     public static <T> List<T> randomSubsetOf(int size, T... values) {
-        if (size > values.length) {
-            throw new IllegalArgumentException("Can\'t pick " + size + " random objects from a list of " + values.length + " objects");
-        }
         List<T> list = arrayAsArrayList(values);
-        Collections.shuffle(list, random());
-        return list.subList(0, size);
+        return randomSubsetOf(size, list);
+    }
+
+    /**
+     * Returns a random subset of values (including a potential empty list)
+     */
+    public static <T> List<T> randomSubsetOf(Collection<T> collection) {
+        return randomSubsetOf(randomInt(collection.size() - 1), collection);
+    }
+
+    /**
+     * Returns size random values
+     */
+    public static <T> List<T> randomSubsetOf(int size, Collection<T> collection) {
+        if (size > collection.size()) {
+            throw new IllegalArgumentException("Can\'t pick " + size + " random objects from a collection of " + collection.size() + " objects");
+        }
+        List<T> tempList = new ArrayList<>(collection);
+        Collections.shuffle(tempList, random());
+        return tempList.subList(0, size);
     }
 
     /**
@@ -642,4 +684,32 @@ public abstract class ESTestCase extends LuceneTestCase {
         }
         return elapsed;
     }
+
+    /**
+     * Creates an AnalysisService to test analysis factories and analyzers.
+     */
+    @SafeVarargs
+    public static AnalysisService createAnalysisService(Index index, Settings settings, Consumer<AnalysisModule>... moduleConsumers) throws IOException {
+        Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
+        return createAnalysisService(index, nodeSettings, settings, moduleConsumers);
+    }
+
+    /**
+     * Creates an AnalysisService to test analysis factories and analyzers.
+     */
+    @SafeVarargs
+    public static AnalysisService createAnalysisService(Index index, Settings nodeSettings, Settings settings, Consumer<AnalysisModule>... moduleConsumers) throws IOException {
+        Settings indexSettings = settingsBuilder().put(settings)
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .build();
+        Environment env = new Environment(nodeSettings);
+        AnalysisModule analysisModule = new AnalysisModule(env);
+        for (Consumer<AnalysisModule> consumer : moduleConsumers) {
+            consumer.accept(analysisModule);
+        }
+        SettingsModule settingsModule = new SettingsModule(nodeSettings);
+        settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
+        final AnalysisService analysisService = analysisModule.buildRegistry().build(IndexSettingsModule.newIndexSettings(index, indexSettings));
+        return analysisService;
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java b/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java
index 64719f0f9de..f76ae7b4b56 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java
@@ -20,6 +20,7 @@ package org.elasticsearch.test;
 
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.plugins.Plugin;
 
@@ -31,12 +32,15 @@ public final class InternalSettingsPlugin extends Plugin {
 
     @Override
     public String description() {
-        return "a plugin that allows to set values for internal settings which are can't be set via the ordinary API without this pluging installed";
+        return "a plugin that allows to set values for internal settings which are can't be set via the ordinary API without this plugin installed";
     }
 
-    public static final Setting<Integer> VERSION_CREATED = Setting.intSetting("index.version.created", 0, false, Setting.Scope.INDEX);
-    public static final Setting<Boolean> MERGE_ENABLED = Setting.boolSetting("index.merge.enabled", true, false, Setting.Scope.INDEX);
-    public static final Setting<Long> INDEX_CREATION_DATE_SETTING = Setting.longSetting(IndexMetaData.SETTING_CREATION_DATE, -1, -1, false, Setting.Scope.INDEX);
+    public static final Setting<Integer> VERSION_CREATED =
+        Setting.intSetting("index.version.created", 0, Property.IndexScope, Property.NodeScope);
+    public static final Setting<Boolean> MERGE_ENABLED =
+        Setting.boolSetting("index.merge.enabled", true, Property.IndexScope, Property.NodeScope);
+    public static final Setting<Long> INDEX_CREATION_DATE_SETTING =
+        Setting.longSetting(IndexMetaData.SETTING_CREATION_DATE, -1, -1, Property.IndexScope, Property.NodeScope);
 
     public void onModule(SettingsModule module) {
         module.registerSetting(VERSION_CREATED);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 4de848ac2d4..736956d249d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -30,22 +30,22 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
+import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
 import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.OperationRouting;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
-import org.elasticsearch.cluster.service.InternalClusterService;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.breaker.CircuitBreaker;
@@ -66,7 +66,7 @@ import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.http.HttpServerTransport;
-import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.CommitStats;
 import org.elasticsearch.index.engine.Engine;
@@ -418,14 +418,6 @@ public final class InternalTestCluster extends TestCluster {
             builder.put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_TYPE_SETTING.getKey(), "noop");
         }
 
-        if (random.nextBoolean()) {
-            builder.put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), random.nextBoolean() ? IndexModule.INDEX_QUERY_CACHE : IndexModule.NONE_QUERY_CACHE);
-        }
-
-        if (random.nextBoolean()) {
-            builder.put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), random.nextBoolean());
-        }
-
         if (random.nextBoolean()) {
             if (random.nextInt(10) == 0) { // do something crazy slow here
                 builder.put(IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING.getKey(), new ByteSizeValue(RandomInts.randomIntBetween(random, 1, 10), ByteSizeUnit.MB));
@@ -456,9 +448,6 @@ public final class InternalTestCluster extends TestCluster {
             builder.put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getKey(), TimeValue.timeValueMillis(RandomInts.randomIntBetween(random, 750, 10000000)));
         }
 
-        // always default delayed allocation to 0 to make sure we have tests are not delayed
-        builder.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), 0);
-
         return builder.build();
     }
 
@@ -590,7 +579,7 @@ public final class InternalTestCluster extends TestCluster {
                 .put(Environment.PATH_HOME_SETTING.getKey(), baseDir) // allow overriding path.home
                 .put(settings)
                 .put("node.name", name)
-                .put(InternalClusterService.NODE_ID_SEED_SETTING.getKey(), seed)
+                .put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), seed)
                 .build();
         MockNode node = new MockNode(finalSettings, version, plugins);
         return new NodeAndClient(name, node);
@@ -837,8 +826,8 @@ public final class InternalTestCluster extends TestCluster {
                     IOUtils.rm(nodeEnv.nodeDataPaths());
                 }
             }
-            final long newIdSeed = InternalClusterService.NODE_ID_SEED_SETTING.get(node.settings()) + 1; // use a new seed to make sure we have new node id
-            Settings finalSettings = Settings.builder().put(node.settings()).put(newSettings).put(InternalClusterService.NODE_ID_SEED_SETTING.getKey(), newIdSeed).build();
+            final long newIdSeed = DiscoveryNodeService.NODE_ID_SEED_SETTING.get(node.settings()) + 1; // use a new seed to make sure we have new node id
+            Settings finalSettings = Settings.builder().put(node.settings()).put(newSettings).put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), newIdSeed).build();
             Collection<Class<? extends Plugin>> plugins = node.getPlugins();
             Version version = node.getVersion();
             node = new MockNode(finalSettings, version, plugins);
@@ -1697,7 +1686,7 @@ public final class InternalTestCluster extends TestCluster {
         }
     }
 
-    synchronized String routingKeyForShard(String index, String type, int shard, Random random) {
+    synchronized String routingKeyForShard(Index index, String type, int shard, Random random) {
         assertThat(shard, greaterThanOrEqualTo(0));
         assertThat(shard, greaterThanOrEqualTo(0));
         for (NodeAndClient n : nodes.values()) {
@@ -1710,7 +1699,7 @@ public final class InternalTestCluster extends TestCluster {
                 OperationRouting operationRouting = getInstanceFromNode(OperationRouting.class, node);
                 while (true) {
                     String routing = RandomStrings.randomAsciiOfLength(random, 10);
-                    final int targetShard = operationRouting.indexShards(clusterService.state(), index, type, null, routing).shardId().getId();
+                    final int targetShard = operationRouting.indexShards(clusterService.state(), index.getName(), type, null, routing).shardId().getId();
                     if (shard == targetShard) {
                         return routing;
                     }
@@ -1849,7 +1838,8 @@ public final class InternalTestCluster extends TestCluster {
                 }
 
                 NodeService nodeService = getInstanceFromNode(NodeService.class, nodeAndClient.node);
-                NodeStats stats = nodeService.stats(CommonStatsFlags.ALL, false, false, false, false, false, false, false, false, false, false);
+                CommonStatsFlags flags = new CommonStatsFlags(Flag.FieldData, Flag.QueryCache, Flag.Segments);
+                NodeStats stats = nodeService.stats(flags, false, false, false, false, false, false, false, false, false, false, false);
                 assertThat("Fielddata size must be 0 on node: " + stats.getNode(), stats.getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
                 assertThat("Query cache size must be 0 on node: " + stats.getNode(), stats.getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0L));
                 assertThat("FixedBitSet cache size must be 0 on node: " + stats.getNode(), stats.getIndices().getSegments().getBitsetMemoryInBytes(), equalTo(0L));
diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java b/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
index 13f533a583e..f17fe024f14 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
@@ -22,6 +22,7 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.index.Index;
@@ -63,7 +64,7 @@ public final class MockIndexEventListener {
         /**
          * For tests to pass in to fail on listener invocation
          */
-        public static final Setting<Boolean> INDEX_FAIL = Setting.boolSetting("index.fail", false, false, Setting.Scope.INDEX);
+        public static final Setting<Boolean> INDEX_FAIL = Setting.boolSetting("index.fail", false, Property.IndexScope);
         public void onModule(SettingsModule module) {
             module.registerSetting(INDEX_FAIL);
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
index 2c8a4c7e458..891353ef589 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
@@ -41,6 +41,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
+import org.elasticsearch.index.percolator.PercolatorQueryCache;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexShard;
@@ -78,6 +79,7 @@ public class TestSearchContext extends SearchContext {
     final IndexService indexService;
     final IndexFieldDataService indexFieldDataService;
     final BitsetFilterCache fixedBitSetFilterCache;
+    final PercolatorQueryCache percolatorQueryCache;
     final ThreadPool threadPool;
     final Map<Class<?>, Collector> queryCollectors = new HashMap<>();
     final IndexShard indexShard;
@@ -105,6 +107,7 @@ public class TestSearchContext extends SearchContext {
         this.indexService = indexService;
         this.indexFieldDataService = indexService.fieldData();
         this.fixedBitSetFilterCache = indexService.cache().bitsetFilterCache();
+        this.percolatorQueryCache = indexService.cache().getPercolatorQueryCache();
         this.threadPool = threadPool;
         this.indexShard = indexService.getShardOrNull(0);
         this.scriptService = scriptService;
@@ -119,6 +122,7 @@ public class TestSearchContext extends SearchContext {
         this.indexFieldDataService = null;
         this.threadPool = null;
         this.fixedBitSetFilterCache = null;
+        this.percolatorQueryCache = null;
         this.indexShard = null;
         scriptService = null;
         this.queryShardContext = queryShardContext;
@@ -330,6 +334,11 @@ public class TestSearchContext extends SearchContext {
         return indexFieldDataService;
     }
 
+    @Override
+    public PercolatorQueryCache percolatorQueryCache() {
+        return percolatorQueryCache;
+    }
+
     @Override
     public long timeoutInMillis() {
         return 0;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/cluster/NoopClusterService.java b/test/framework/src/main/java/org/elasticsearch/test/cluster/NoopClusterService.java
deleted file mode 100644
index 99ba809c144..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/cluster/NoopClusterService.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.cluster;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
-import org.elasticsearch.cluster.ClusterStateTaskConfig;
-import org.elasticsearch.cluster.ClusterStateTaskExecutor;
-import org.elasticsearch.cluster.ClusterStateTaskListener;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
-import org.elasticsearch.cluster.LocalNodeMasterListener;
-import org.elasticsearch.cluster.TimeoutClusterStateListener;
-import org.elasticsearch.cluster.block.ClusterBlock;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.OperationRouting;
-import org.elasticsearch.cluster.service.PendingClusterTask;
-import org.elasticsearch.common.component.Lifecycle;
-import org.elasticsearch.common.component.LifecycleListener;
-import org.elasticsearch.common.transport.DummyTransportAddress;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.tasks.TaskManager;
-
-import java.util.List;
-
-public class NoopClusterService implements ClusterService {
-
-    final ClusterState state;
-
-    public NoopClusterService() {
-        this(ClusterState.builder(new ClusterName("noop")).build());
-    }
-
-    public NoopClusterService(ClusterState state) {
-        if (state.getNodes().size() == 0) {
-            state = ClusterState.builder(state).nodes(
-                    DiscoveryNodes.builder()
-                            .put(new DiscoveryNode("noop_id", DummyTransportAddress.INSTANCE, Version.CURRENT))
-                            .localNodeId("noop_id")).build();
-        }
-
-        assert state.getNodes().localNode() != null;
-        this.state = state;
-
-    }
-
-    @Override
-    public DiscoveryNode localNode() {
-        return state.getNodes().localNode();
-    }
-
-    @Override
-    public ClusterState state() {
-        return state;
-    }
-
-    @Override
-    public void addInitialStateBlock(ClusterBlock block) throws IllegalStateException {
-
-    }
-
-    @Override
-    public void removeInitialStateBlock(ClusterBlock block) throws IllegalStateException {
-
-    }
-
-    @Override
-    public void removeInitialStateBlock(int blockId) throws IllegalStateException {
-
-    }
-
-    @Override
-    public OperationRouting operationRouting() {
-        return null;
-    }
-
-    @Override
-    public void addFirst(ClusterStateListener listener) {
-
-    }
-
-    @Override
-    public void addLast(ClusterStateListener listener) {
-
-    }
-
-    @Override
-    public void add(ClusterStateListener listener) {
-
-    }
-
-    @Override
-    public void remove(ClusterStateListener listener) {
-
-    }
-
-    @Override
-    public void add(LocalNodeMasterListener listener) {
-
-    }
-
-    @Override
-    public void remove(LocalNodeMasterListener listener) {
-
-    }
-
-    @Override
-    public void add(TimeValue timeout, TimeoutClusterStateListener listener) {
-
-    }
-
-    @Override
-    public void submitStateUpdateTask(String source, ClusterStateUpdateTask updateTask) {
-
-    }
-
-    @Override
-    public <T> void submitStateUpdateTask(String source, T task, ClusterStateTaskConfig config, ClusterStateTaskExecutor<T> executor, ClusterStateTaskListener listener) {
-
-    }
-
-    @Override
-    public List<PendingClusterTask> pendingTasks() {
-        return null;
-    }
-
-    @Override
-    public int numberOfPendingTasks() {
-        return 0;
-    }
-
-    @Override
-    public TimeValue getMaxTaskWaitTime() {
-        return TimeValue.timeValueMillis(0);
-    }
-
-    @Override
-    public TaskManager getTaskManager() {
-        return null;
-    }
-
-    @Override
-    public Lifecycle.State lifecycleState() {
-        return null;
-    }
-
-    @Override
-    public void addLifecycleListener(LifecycleListener listener) {
-
-    }
-
-    @Override
-    public void removeLifecycleListener(LifecycleListener listener) {
-
-    }
-
-    @Override
-    public ClusterService start() {
-        return null;
-    }
-
-    @Override
-    public ClusterService stop() {
-        return null;
-    }
-
-    @Override
-    public void close() {
-
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/cluster/TestClusterService.java b/test/framework/src/main/java/org/elasticsearch/test/cluster/TestClusterService.java
deleted file mode 100644
index 3b1082cae44..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/cluster/TestClusterService.java
+++ /dev/null
@@ -1,315 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.cluster;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
-import org.elasticsearch.cluster.ClusterStateTaskConfig;
-import org.elasticsearch.cluster.ClusterStateTaskExecutor;
-import org.elasticsearch.cluster.ClusterStateTaskListener;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
-import org.elasticsearch.cluster.LocalNodeMasterListener;
-import org.elasticsearch.cluster.TimeoutClusterStateListener;
-import org.elasticsearch.cluster.block.ClusterBlock;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.OperationRouting;
-import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
-import org.elasticsearch.cluster.service.PendingClusterTask;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.component.Lifecycle;
-import org.elasticsearch.common.component.LifecycleListener;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
-import org.elasticsearch.common.util.concurrent.FutureUtils;
-import org.elasticsearch.tasks.TaskManager;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.ScheduledFuture;
-
-/** a class that simulate simple cluster service features, like state storage and listeners */
-public class TestClusterService implements ClusterService {
-
-    volatile ClusterState state;
-    private volatile TaskManager taskManager;
-    private final List<ClusterStateListener> listeners = new CopyOnWriteArrayList<>();
-    private final Queue<NotifyTimeout> onGoingTimeouts = ConcurrentCollections.newQueue();
-    private final ThreadPool threadPool;
-    private final ESLogger logger = Loggers.getLogger(getClass(), Settings.EMPTY);
-    private final OperationRouting operationRouting = new OperationRouting(Settings.Builder.EMPTY_SETTINGS, new AwarenessAllocationDecider());
-
-    public TestClusterService() {
-        this(ClusterState.builder(new ClusterName("test")).build());
-    }
-
-    public TestClusterService(ThreadPool threadPool) {
-        this(ClusterState.builder(new ClusterName("test")).build(), threadPool);
-        taskManager = new TaskManager(Settings.EMPTY);
-    }
-
-    public TestClusterService(ThreadPool threadPool, TransportService transportService) {
-        this(ClusterState.builder(new ClusterName("test")).build(), threadPool);
-        taskManager = transportService.getTaskManager();
-    }
-
-    public TestClusterService(ClusterState state) {
-        this(state, null);
-    }
-
-    public TestClusterService(ClusterState state, @Nullable ThreadPool threadPool) {
-        if (state.getNodes().size() == 0) {
-            state = ClusterState.builder(state).nodes(
-                    DiscoveryNodes.builder()
-                            .put(new DiscoveryNode("test_node", DummyTransportAddress.INSTANCE, Version.CURRENT))
-                            .localNodeId("test_node")).build();
-        }
-
-        assert state.getNodes().localNode() != null;
-        this.state = state;
-        this.threadPool = threadPool;
-
-    }
-
-
-    /** set the current state and trigger any registered listeners about the change, mimicking an update task */
-    synchronized public ClusterState setState(ClusterState state) {
-        assert state.getNodes().localNode() != null;
-        // make sure we have a version increment
-        state = ClusterState.builder(state).version(this.state.version() + 1).build();
-        return setStateAndNotifyListeners(state);
-    }
-
-    private ClusterState setStateAndNotifyListeners(ClusterState state) {
-        ClusterChangedEvent event = new ClusterChangedEvent("test", state, this.state);
-        this.state = state;
-        for (ClusterStateListener listener : listeners) {
-            listener.clusterChanged(event);
-        }
-        return state;
-    }
-
-    /** set the current state and trigger any registered listeners about the change */
-    public ClusterState setState(ClusterState.Builder state) {
-        return setState(state.build());
-    }
-
-    @Override
-    public DiscoveryNode localNode() {
-        return state.getNodes().localNode();
-    }
-
-    @Override
-    public ClusterState state() {
-        return state;
-    }
-
-    @Override
-    public void addInitialStateBlock(ClusterBlock block) throws IllegalStateException {
-        throw new UnsupportedOperationException();
-
-    }
-
-    @Override
-    public void removeInitialStateBlock(ClusterBlock block) throws IllegalStateException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void removeInitialStateBlock(int blockId) throws IllegalStateException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public OperationRouting operationRouting() {
-        return operationRouting;
-    }
-
-    @Override
-    public void addFirst(ClusterStateListener listener) {
-        listeners.add(0, listener);
-    }
-
-    @Override
-    public void addLast(ClusterStateListener listener) {
-        listeners.add(listener);
-    }
-
-    @Override
-    public void add(ClusterStateListener listener) {
-        listeners.add(listener);
-    }
-
-    @Override
-    public void remove(ClusterStateListener listener) {
-        listeners.remove(listener);
-        for (Iterator<NotifyTimeout> it = onGoingTimeouts.iterator(); it.hasNext(); ) {
-            NotifyTimeout timeout = it.next();
-            if (timeout.listener.equals(listener)) {
-                timeout.cancel();
-                it.remove();
-            }
-        }
-    }
-
-    @Override
-    public void add(LocalNodeMasterListener listener) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void remove(LocalNodeMasterListener listener) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void add(final TimeValue timeout, final TimeoutClusterStateListener listener) {
-        if (threadPool == null) {
-            throw new UnsupportedOperationException("TestClusterService wasn't initialized with a thread pool");
-        }
-        if (timeout != null) {
-            NotifyTimeout notifyTimeout = new NotifyTimeout(listener, timeout);
-            notifyTimeout.future = threadPool.schedule(timeout, ThreadPool.Names.GENERIC, notifyTimeout);
-            onGoingTimeouts.add(notifyTimeout);
-        }
-        listeners.add(listener);
-        listener.postAdded();
-    }
-
-    @Override
-    public void submitStateUpdateTask(String source, ClusterStateUpdateTask updateTask) {
-        submitStateUpdateTask(source, null, updateTask, updateTask, updateTask);
-    }
-
-    @Override
-    synchronized public <T> void submitStateUpdateTask(String source, T task, ClusterStateTaskConfig config, ClusterStateTaskExecutor<T> executor, ClusterStateTaskListener listener) {
-        logger.debug("processing [{}]", source);
-        if (state().nodes().localNodeMaster() == false && executor.runOnlyOnMaster()) {
-            listener.onNoLongerMaster(source);
-            logger.debug("failed [{}], no longer master", source);
-            return;
-        }
-        ClusterStateTaskExecutor.BatchResult<T> batchResult;
-        ClusterState previousClusterState = state;
-        try {
-            batchResult = executor.execute(previousClusterState, Arrays.asList(task));
-        } catch (Exception e) {
-            batchResult = ClusterStateTaskExecutor.BatchResult.<T>builder().failure(task, e).build(previousClusterState);
-        }
-
-        batchResult.executionResults.get(task).handle(
-                () -> {},
-                ex -> listener.onFailure(source, new ElasticsearchException("failed to process cluster state update task [" + source + "]", ex))
-        );
-
-        setStateAndNotifyListeners(batchResult.resultingState);
-        listener.clusterStateProcessed(source, previousClusterState, batchResult.resultingState);
-        logger.debug("finished [{}]", source);
-
-    }
-
-    @Override
-    public TimeValue getMaxTaskWaitTime() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public TaskManager getTaskManager() {
-        return taskManager;
-    }
-
-    @Override
-    public List<PendingClusterTask> pendingTasks() {
-        throw new UnsupportedOperationException();
-
-    }
-
-    @Override
-    public int numberOfPendingTasks() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public Lifecycle.State lifecycleState() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void addLifecycleListener(LifecycleListener listener) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void removeLifecycleListener(LifecycleListener listener) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public ClusterService start() throws ElasticsearchException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public ClusterService stop() throws ElasticsearchException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void close() throws ElasticsearchException {
-        throw new UnsupportedOperationException();
-    }
-
-    class NotifyTimeout implements Runnable {
-        final TimeoutClusterStateListener listener;
-        final TimeValue timeout;
-        volatile ScheduledFuture future;
-
-        NotifyTimeout(TimeoutClusterStateListener listener, TimeValue timeout) {
-            this.listener = listener;
-            this.timeout = timeout;
-        }
-
-        public void cancel() {
-            FutureUtils.cancel(future);
-        }
-
-        @Override
-        public void run() {
-            if (future != null && future.isCancelled()) {
-                return;
-            }
-            listener.onTimeout(this.timeout);
-            // note, we rely on the listener to remove itself in case of timeout if needed
-        }
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java
index e318843e84f..cbcb9766943 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java
@@ -18,9 +18,9 @@
  */
 package org.elasticsearch.test.disruption;
 
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.InternalTestCluster;
@@ -56,7 +56,7 @@ public class BlockClusterStateProcessing extends SingleNodeDisruption {
         }
         logger.info("delaying cluster state updates on node [{}]", disruptionNodeCopy);
         boolean success = disruptionLatch.compareAndSet(null, new CountDownLatch(1));
-        assert success : "startDisrupting called without waiting on stopDistrupting to complete";
+        assert success : "startDisrupting called without waiting on stopDisrupting to complete";
         final CountDownLatch started = new CountDownLatch(1);
         clusterService.submitStateUpdateTask("service_disruption_block", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java
index b9c663686b1..be0b69a8e8b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java
@@ -18,9 +18,9 @@
  */
 package org.elasticsearch.test.disruption;
 
-import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.InternalTestCluster;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java
index ddccfe88e38..bf32b6b8575 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java
@@ -31,6 +31,7 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineConfig;
@@ -55,11 +56,13 @@ public final class MockEngineSupport {
      * Allows tests to wrap an index reader randomly with a given ratio. This is disabled by default ie. <tt>0.0d</tt> since reader wrapping is insanely
      * slow if {@link org.apache.lucene.index.AssertingDirectoryReader} is used.
      */
-    public static final Setting<Double> WRAP_READER_RATIO = Setting.doubleSetting("index.engine.mock.random.wrap_reader_ratio", 0.0d, 0.0d, false, Setting.Scope.INDEX);
+    public static final Setting<Double> WRAP_READER_RATIO =
+        Setting.doubleSetting("index.engine.mock.random.wrap_reader_ratio", 0.0d, 0.0d, Property.IndexScope);
     /**
      * Allows tests to prevent an engine from being flushed on close ie. to test translog recovery...
      */
-    public static final Setting<Boolean> DISABLE_FLUSH_ON_CLOSE = Setting.boolSetting("index.mock.disable_flush_on_close", false, false, Setting.Scope.INDEX);
+    public static final Setting<Boolean> DISABLE_FLUSH_ON_CLOSE =
+        Setting.boolSetting("index.mock.disable_flush_on_close", false, Property.IndexScope);
 
 
     private final AtomicBoolean closing = new AtomicBoolean(false);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
index cb3bbc7436b..7adf6d2b4b9 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
@@ -144,7 +144,7 @@ public class ElasticsearchAssertions {
     }
 
     /**
-     * Checks that all shard requests of a replicated brodcast request failed due to a cluster block
+     * Checks that all shard requests of a replicated broadcast request failed due to a cluster block
      *
      * @param replicatedBroadcastResponse the response that should only contain failed shard responses
      *
@@ -716,7 +716,7 @@ public class ElasticsearchAssertions {
 
     /**
      * Applies basic assertions on the SearchResponse. This method checks if all shards were successful, if
-     * any of the shards threw an exception and if the response is serializeable.
+     * any of the shards threw an exception and if the response is serializable.
      */
     public static SearchResponse assertSearchResponse(SearchRequestBuilder request) {
         return assertSearchResponse(request.get());
@@ -724,7 +724,7 @@ public class ElasticsearchAssertions {
 
     /**
      * Applies basic assertions on the SearchResponse. This method checks if all shards were successful, if
-     * any of the shards threw an exception and if the response is serializeable.
+     * any of the shards threw an exception and if the response is serializable.
      */
     public static SearchResponse assertSearchResponse(SearchResponse response) {
         assertNoFailures(response);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index 5684717342d..fbc518b136d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -19,14 +19,34 @@
 
 package org.elasticsearch.test.rest;
 
-import com.carrotsearch.randomizedtesting.RandomizedTest;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.client.RestException;
+import org.elasticsearch.test.rest.client.RestResponse;
 import org.elasticsearch.test.rest.parser.RestTestParseException;
 import org.elasticsearch.test.rest.parser.RestTestSuiteParser;
 import org.elasticsearch.test.rest.section.DoSection;
@@ -42,24 +62,11 @@ import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.nio.file.FileSystem;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.sort;
 
 /**
  * Runs the clients test suite against an elasticsearch cluster.
@@ -261,7 +268,6 @@ public abstract class ESRestTestCase extends ESTestCase {
 
     @After
     public void wipeCluster() throws Exception {
-
         // wipe indices
         Map<String, String> deleteIndicesArgs = new HashMap<>();
         deleteIndicesArgs.put("index", "*");
@@ -285,6 +291,30 @@ public abstract class ESRestTestCase extends ESTestCase {
         adminExecutionContext.callApi("snapshot.delete_repository", deleteSnapshotsArgs, Collections.emptyList(), Collections.emptyMap());
     }
 
+    /**
+     * Logs a message if there are still running tasks. The reasoning is that any tasks still running are state the is trying to bleed into
+     * other tests.
+     */
+    @After
+    public void logIfThereAreRunningTasks() throws InterruptedException, IOException, RestException {
+        RestResponse tasks = adminExecutionContext.callApi("tasks.list", emptyMap(), emptyList(), emptyMap());
+        Set<String> runningTasks = runningTasks(tasks);
+        // Ignore the task list API - it doens't count against us
+        runningTasks.remove(ListTasksAction.NAME);
+        runningTasks.remove(ListTasksAction.NAME + "[n]");
+        if (runningTasks.isEmpty()) {
+            return;
+        }
+        List<String> stillRunning = new ArrayList<>(runningTasks);
+        sort(stillRunning);
+        logger.info("There are still tasks running after this test that might break subsequent tests {}.", stillRunning);
+        /*
+         * This isn't a higher level log or outright failure because some of these tasks are run by the cluster in the background. If we
+         * could determine that some tasks are run by the user we'd fail the tests if those tasks were running and ignore any background
+         * tasks.
+         */
+    }
+
     @AfterClass
     public static void close() {
         if (restTestExecutionContext != null) {
@@ -365,4 +395,19 @@ public abstract class ESRestTestCase extends ESTestCase {
             executableSection.execute(restTestExecutionContext);
         }
     }
+
+    @SuppressWarnings("unchecked")
+    public Set<String> runningTasks(RestResponse response) throws IOException {
+        Set<String> runningTasks = new HashSet<>();
+        Map<String, Object> nodes = (Map<String, Object>) response.evaluate("nodes");
+        for (Map.Entry<String, Object> node : nodes.entrySet()) {
+            Map<String, Object> nodeInfo = (Map<String, Object>) node.getValue();
+            Map<String, Object> nodeTasks = (Map<String, Object>) nodeInfo.get("tasks");
+            for (Map.Entry<String, Object> taskAndName : nodeTasks.entrySet()) {
+                Map<String, Object> task = (Map<String, Object>) taskAndName.getValue();
+                runningTasks.add(task.get("action").toString());
+            }
+        }
+        return runningTasks;
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java
index e5597713570..e798fd8c8ab 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java
@@ -151,8 +151,7 @@ public class RestClient implements Closeable {
 
         HttpRequestBuilder httpRequestBuilder = callApiBuilder(apiName, requestParams, body);
         for (Map.Entry<String, String> header : headers.entrySet()) {
-            logger.error("Adding header " + header.getKey());
-            logger.error(" with value " + header.getValue());
+            logger.error("Adding header {}\n with value {}", header.getKey(), header.getValue());
             httpRequestBuilder.addHeader(header.getKey(), header.getValue());
         }
         logger.debug("calling api [{}]", apiName);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java
index 6a484e9ae69..79f7502fb27 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java
@@ -114,9 +114,10 @@ public class HttpRequestBuilder {
         for (String pathPart : path) {
             try {
                 finalPath.append('/');
-                URI uri = new URI(null, null, null, -1, pathPart, null, null);
+                // We append "/" to the path part to handle parts that start with - or other invalid characters
+                URI uri = new URI(null, null, null, -1, "/" + pathPart, null, null);
                 //manually escape any slash that each part may contain
-                finalPath.append(uri.getRawPath().replaceAll("/", "%2F"));
+                finalPath.append(uri.getRawPath().substring(1).replaceAll("/", "%2F"));
             } catch(URISyntaxException e) {
                 throw new RuntimeException("unable to build uri", e);
             }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpResponse.java
index 9945edbefa9..37fc163ac61 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpResponse.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpResponse.java
@@ -61,7 +61,7 @@ public class HttpResponse {
                 try {
                     httpResponse.close();
                 } catch (IOException e) {
-                    logger.error(e.getMessage(), e);
+                    logger.error("Failed closing response", e);
                 }
             }
         } else {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java b/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java
index 0f51f72e8e5..66ba1528b90 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java
@@ -34,7 +34,7 @@ import java.util.List;
  */
 public final class Features {
 
-    private static final List<String> SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting", "headers");
+    private static final List<String> SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting", "headers", "yaml");
 
     private Features() {
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java
index c945a308363..7d17746d54f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java
@@ -23,7 +23,6 @@ import com.carrotsearch.randomizedtesting.SeedUtils;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 
 import org.apache.lucene.index.CheckIndex;
-import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.store.BaseDirectoryWrapper;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.LockFactory;
@@ -32,13 +31,13 @@ import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.StoreRateLimiting;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestRuleMarkFailure;
-import org.elasticsearch.cluster.metadata.AliasOrIndex;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
@@ -57,17 +56,20 @@ import java.io.PrintStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.Random;
-import java.util.Set;
 
 public class MockFSDirectoryService extends FsDirectoryService {
 
-    public static final Setting<Double> RANDOM_IO_EXCEPTION_RATE_ON_OPEN_SETTING = Setting.doubleSetting("index.store.mock.random.io_exception_rate_on_open", 0.0d,  0.0d, false, Setting.Scope.INDEX);
-    public static final Setting<Double> RANDOM_IO_EXCEPTION_RATE_SETTING = Setting.doubleSetting("index.store.mock.random.io_exception_rate", 0.0d,  0.0d, false, Setting.Scope.INDEX);
-    public static final Setting<Boolean> RANDOM_PREVENT_DOUBLE_WRITE_SETTING = Setting.boolSetting("index.store.mock.random.prevent_double_write", true, false, Setting.Scope.INDEX);// true is default in MDW
-    public static final Setting<Boolean> RANDOM_NO_DELETE_OPEN_FILE_SETTING = Setting.boolSetting("index.store.mock.random.no_delete_open_file", true, false, Setting.Scope.INDEX);// true is default in MDW
-    public static final Setting<Boolean> CRASH_INDEX_SETTING = Setting.boolSetting("index.store.mock.random.crash_index", true, false, Setting.Scope.INDEX);// true is default in MDW
+    public static final Setting<Double> RANDOM_IO_EXCEPTION_RATE_ON_OPEN_SETTING =
+        Setting.doubleSetting("index.store.mock.random.io_exception_rate_on_open", 0.0d,  0.0d, Property.IndexScope, Property.NodeScope);
+    public static final Setting<Double> RANDOM_IO_EXCEPTION_RATE_SETTING =
+        Setting.doubleSetting("index.store.mock.random.io_exception_rate", 0.0d,  0.0d, Property.IndexScope, Property.NodeScope);
+    public static final Setting<Boolean> RANDOM_PREVENT_DOUBLE_WRITE_SETTING =
+        Setting.boolSetting("index.store.mock.random.prevent_double_write", true, Property.IndexScope, Property.NodeScope);// true is default in MDW
+    public static final Setting<Boolean> RANDOM_NO_DELETE_OPEN_FILE_SETTING =
+        Setting.boolSetting("index.store.mock.random.no_delete_open_file", true, Property.IndexScope, Property.NodeScope);// true is default in MDW
+    public static final Setting<Boolean> CRASH_INDEX_SETTING =
+        Setting.boolSetting("index.store.mock.random.crash_index", true, Property.IndexScope, Property.NodeScope);// true is default in MDW
 
     private final FsDirectoryService delegateService;
     private final Random random;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java
index 80251d54951..d44cf60e9e3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java
@@ -23,6 +23,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.index.IndexModule;
@@ -44,7 +45,8 @@ import java.util.Map;
 
 public class MockFSIndexStore extends IndexStore {
 
-    public static final Setting<Boolean> INDEX_CHECK_INDEX_ON_CLOSE_SETTING = Setting.boolSetting("index.store.mock.check_index_on_close", true, false, Setting.Scope.INDEX);
+    public static final Setting<Boolean> INDEX_CHECK_INDEX_ON_CLOSE_SETTING =
+        Setting.boolSetting("index.store.mock.check_index_on_close", true, Property.IndexScope, Property.NodeScope);
 
     public static class TestPlugin extends Plugin {
         @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
index 4c48f990d6a..fc090e151a3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.test.tasks;
 
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskManager;
@@ -33,7 +34,8 @@ import java.util.concurrent.CopyOnWriteArrayList;
  */
 public class MockTaskManager extends TaskManager {
 
-    public static final Setting<Boolean> USE_MOCK_TASK_MANAGER_SETTING = Setting.boolSetting("tests.mock.taskmanager.enabled", false, false, Setting.Scope.CLUSTER);
+    public static final Setting<Boolean> USE_MOCK_TASK_MANAGER_SETTING =
+        Setting.boolSetting("tests.mock.taskmanager.enabled", false, Property.NodeScope);
 
     private final Collection<MockTaskManagerListener> listeners = new CopyOnWriteArrayList<>();
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java
index fb310239155..322882a7b3c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.plugins.Plugin;
@@ -67,10 +68,12 @@ public class AssertingLocalTransport extends LocalTransport {
         }
     }
 
-    public static final Setting<Version> ASSERTING_TRANSPORT_MIN_VERSION_KEY = new Setting<>("transport.asserting.version.min",
-            Integer.toString(Version.CURRENT.minimumCompatibilityVersion().id), (s) -> Version.fromId(Integer.parseInt(s)), false, Setting.Scope.CLUSTER);
-    public static final Setting<Version> ASSERTING_TRANSPORT_MAX_VERSION_KEY = new Setting<>("transport.asserting.version.max",
-        Integer.toString(Version.CURRENT.id), (s) -> Version.fromId(Integer.parseInt(s)), false, Setting.Scope.CLUSTER);
+    public static final Setting<Version> ASSERTING_TRANSPORT_MIN_VERSION_KEY =
+        new Setting<>("transport.asserting.version.min", Integer.toString(Version.CURRENT.minimumCompatibilityVersion().id),
+            (s) -> Version.fromId(Integer.parseInt(s)), Property.NodeScope);
+    public static final Setting<Version> ASSERTING_TRANSPORT_MAX_VERSION_KEY =
+        new Setting<>("transport.asserting.version.max", Integer.toString(Version.CURRENT.id),
+            (s) -> Version.fromId(Integer.parseInt(s)), Property.NodeScope);
     private final Random random;
     private final Version minVersion;
     private final Version maxVersion;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
index 03745183240..f5fd5123847 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
@@ -99,22 +99,22 @@ public class MockTransportService extends TransportService {
     public static MockTransportService local(Settings settings, Version version, ThreadPool threadPool) {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
         Transport transport = new LocalTransport(settings, threadPool, version, namedWriteableRegistry);
-        return new MockTransportService(settings, transport, threadPool, namedWriteableRegistry);
+        return new MockTransportService(settings, transport, threadPool);
     }
 
     public static MockTransportService nettyFromThreadPool(Settings settings, Version version, ThreadPool threadPool) {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
         Transport transport = new NettyTransport(settings, threadPool, new NetworkService(settings), BigArrays.NON_RECYCLING_INSTANCE,
                 version, namedWriteableRegistry);
-        return new MockTransportService(Settings.EMPTY, transport, threadPool, namedWriteableRegistry);
+        return new MockTransportService(Settings.EMPTY, transport, threadPool);
     }
 
 
     private final Transport original;
 
     @Inject
-    public MockTransportService(Settings settings, Transport transport, ThreadPool threadPool, NamedWriteableRegistry namedWriteableRegistry) {
-        super(settings, new LookupTestTransport(transport), threadPool, namedWriteableRegistry);
+    public MockTransportService(Settings settings, Transport transport, ThreadPool threadPool) {
+        super(settings, new LookupTestTransport(transport), threadPool);
         this.original = transport;
     }
 
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java
index ea1929a55b0..97683b620f0 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java
@@ -46,14 +46,14 @@ public class VersionUtilsTests extends ESTestCase {
         assertTrue(got.onOrBefore(Version.CURRENT));
 
         // sub range
-        got = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0);
+        got = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0_alpha1);
         assertTrue(got.onOrAfter(Version.V_2_0_0));
-        assertTrue(got.onOrBefore(Version.V_5_0_0));
+        assertTrue(got.onOrBefore(Version.V_5_0_0_alpha1));
 
         // unbounded lower
-        got = VersionUtils.randomVersionBetween(random(), null, Version.V_5_0_0);
+        got = VersionUtils.randomVersionBetween(random(), null, Version.V_5_0_0_alpha1);
         assertTrue(got.onOrAfter(VersionUtils.getFirstVersion()));
-        assertTrue(got.onOrBefore(Version.V_5_0_0));
+        assertTrue(got.onOrBefore(Version.V_5_0_0_alpha1));
         got = VersionUtils.randomVersionBetween(random(), null, VersionUtils.allVersions().get(0));
         assertTrue(got.onOrAfter(VersionUtils.getFirstVersion()));
         assertTrue(got.onOrBefore(VersionUtils.allVersions().get(0)));
@@ -71,8 +71,8 @@ public class VersionUtilsTests extends ESTestCase {
         assertEquals(got, VersionUtils.getFirstVersion());
         got = VersionUtils.randomVersionBetween(random(), Version.CURRENT, Version.CURRENT);
         assertEquals(got, Version.CURRENT);
-        got = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, Version.V_5_0_0);
-        assertEquals(got, Version.V_5_0_0);
+        got = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0_alpha1, Version.V_5_0_0_alpha1);
+        assertEquals(got, Version.V_5_0_0_alpha1);
 
         // implicit range of one
         got = VersionUtils.randomVersionBetween(random(), null, VersionUtils.getFirstVersion());
diff --git a/core/src/main/java/org/elasticsearch/percolator/PercolatorIndex.java b/test/logger-usage/build.gradle
similarity index 60%
rename from core/src/main/java/org/elasticsearch/percolator/PercolatorIndex.java
rename to test/logger-usage/build.gradle
index 6f9a7104834..1a5815cf76e 100644
--- a/core/src/main/java/org/elasticsearch/percolator/PercolatorIndex.java
+++ b/test/logger-usage/build.gradle
@@ -1,3 +1,5 @@
+import org.elasticsearch.gradle.precommit.PrecommitTasks
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -17,21 +19,15 @@
  * under the License.
  */
 
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.index.mapper.ParsedDocument;
-
-/**
- * Abstraction on how to index the percolator document.
- */
-interface PercolatorIndex {
-
-    /**
-     * Indexes the document(s) and initializes the PercolateContext
-     *
-     * @param context  Initialized with document related properties for fetch phase.
-     * @param document Document that is percolated. Can contain several documents.
-     * */
-    void prepare(PercolateContext context, ParsedDocument document);
-
+dependencies {
+  compile 'org.ow2.asm:asm-debug-all:5.0.4' // use asm-debug-all as asm-all is broken
+  testCompile "org.elasticsearch.test:framework:${version}"
 }
+
+loggerUsageCheck.enabled = false
+
+forbiddenApisMain.enabled = true // disabled by parent project
+forbiddenApisMain {
+  signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')] // does not depend on core, only jdk signatures
+}
+jarHell.enabled = true // disabled by parent project
\ No newline at end of file
diff --git a/test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/ESLoggerUsageChecker.java b/test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/ESLoggerUsageChecker.java
new file mode 100644
index 00000000000..25d4052c162
--- /dev/null
+++ b/test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/ESLoggerUsageChecker.java
@@ -0,0 +1,459 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.loggerusage;
+
+import org.objectweb.asm.AnnotationVisitor;
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.Type;
+import org.objectweb.asm.tree.AbstractInsnNode;
+import org.objectweb.asm.tree.IntInsnNode;
+import org.objectweb.asm.tree.LdcInsnNode;
+import org.objectweb.asm.tree.LineNumberNode;
+import org.objectweb.asm.tree.MethodInsnNode;
+import org.objectweb.asm.tree.MethodNode;
+import org.objectweb.asm.tree.TypeInsnNode;
+import org.objectweb.asm.tree.analysis.Analyzer;
+import org.objectweb.asm.tree.analysis.AnalyzerException;
+import org.objectweb.asm.tree.analysis.BasicInterpreter;
+import org.objectweb.asm.tree.analysis.BasicValue;
+import org.objectweb.asm.tree.analysis.Frame;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
+
+public class ESLoggerUsageChecker {
+    public static final String LOGGER_CLASS = "org.elasticsearch.common.logging.ESLogger";
+    public static final String THROWABLE_CLASS = "java.lang.Throwable";
+    public static final List<String> LOGGER_METHODS = Arrays.asList("trace", "debug", "info", "warn", "error");
+    public static final String IGNORE_CHECKS_ANNOTATION = "org.elasticsearch.common.SuppressLoggerChecks";
+
+    @SuppressForbidden(reason = "command line tool")
+    public static void main(String... args) throws Exception {
+        System.out.println("checking for wrong usages of ESLogger...");
+        boolean[] wrongUsageFound = new boolean[1];
+        checkLoggerUsage(wrongLoggerUsage -> {
+            System.err.println(wrongLoggerUsage.getErrorLines());
+            wrongUsageFound[0] = true;
+        }, args);
+        if (wrongUsageFound[0]) {
+            throw new Exception("Wrong logger usages found");
+        } else {
+            System.out.println("No wrong usages found");
+        }
+    }
+
+    private static void checkLoggerUsage(Consumer<WrongLoggerUsage> wrongUsageCallback, String... classDirectories)
+        throws IOException {
+        for (String classDirectory : classDirectories) {
+            Path root = Paths.get(classDirectory);
+            if (Files.isDirectory(root) == false) {
+                throw new IllegalArgumentException(root + " should be an existing directory");
+            }
+            Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                    if (Files.isRegularFile(file) && file.endsWith(".class")) {
+                        try (InputStream in = Files.newInputStream(file)) {
+                            ESLoggerUsageChecker.check(wrongUsageCallback, in);
+                        }
+                    }
+                    return super.visitFile(file, attrs);
+                }
+            });
+        }
+    }
+
+    public static void check(Consumer<WrongLoggerUsage> wrongUsageCallback, InputStream inputStream) throws IOException {
+        check(wrongUsageCallback, inputStream, s -> true);
+    }
+
+    // used by tests
+    static void check(Consumer<WrongLoggerUsage> wrongUsageCallback, InputStream inputStream, Predicate<String> methodsToCheck)
+        throws IOException {
+        ClassReader cr = new ClassReader(inputStream);
+        cr.accept(new ClassChecker(wrongUsageCallback, methodsToCheck), 0);
+    }
+
+    public static class WrongLoggerUsage {
+        private final String className;
+        private final String methodName;
+        private final String logMethodName;
+        private final int line;
+        private final String errorMessage;
+
+        public WrongLoggerUsage(String className, String methodName, String logMethodName, int line, String errorMessage) {
+            this.className = className;
+            this.methodName = methodName;
+            this.logMethodName = logMethodName;
+            this.line = line;
+            this.errorMessage = errorMessage;
+        }
+
+        @Override
+        public String toString() {
+            return "WrongLoggerUsage{" +
+                "className='" + className + '\'' +
+                ", methodName='" + methodName + '\'' +
+                ", logMethodName='" + logMethodName + '\'' +
+                ", line=" + line +
+                ", errorMessage='" + errorMessage + '\'' +
+                '}';
+        }
+
+        /**
+         * Returns an error message that has the form of stack traces emitted by {@link Throwable#printStackTrace}
+         */
+        public String getErrorLines() {
+            String fullClassName = Type.getObjectType(className).getClassName();
+            String simpleClassName = fullClassName.substring(fullClassName.lastIndexOf(".") + 1, fullClassName.length());
+            int innerClassIndex = simpleClassName.indexOf("$");
+            if (innerClassIndex > 0) {
+                simpleClassName = simpleClassName.substring(0, innerClassIndex);
+            }
+            simpleClassName = simpleClassName + ".java";
+            StringBuilder sb = new StringBuilder();
+            sb.append("Bad usage of ");
+            sb.append(LOGGER_CLASS).append("#").append(logMethodName);
+            sb.append(": ");
+            sb.append(errorMessage);
+            sb.append("\n\tat ");
+            sb.append(fullClassName);
+            sb.append(".");
+            sb.append(methodName);
+            sb.append("(");
+            sb.append(simpleClassName);
+            sb.append(":");
+            sb.append(line);
+            sb.append(")");
+            return sb.toString();
+        }
+    }
+
+    private static class ClassChecker extends ClassVisitor {
+        private String className;
+        private boolean ignoreChecks;
+        private final Consumer<WrongLoggerUsage> wrongUsageCallback;
+        private final Predicate<String> methodsToCheck;
+
+        public ClassChecker(Consumer<WrongLoggerUsage> wrongUsageCallback, Predicate<String> methodsToCheck) {
+            super(Opcodes.ASM5);
+            this.wrongUsageCallback = wrongUsageCallback;
+            this.methodsToCheck = methodsToCheck;
+        }
+
+        @Override
+        public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) {
+            this.className = name;
+        }
+
+        @Override
+        public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
+            if (IGNORE_CHECKS_ANNOTATION.equals(Type.getType(desc).getClassName())) {
+                ignoreChecks = true;
+            }
+            return super.visitAnnotation(desc, visible);
+        }
+
+        @Override
+        public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) {
+            if (ignoreChecks == false && methodsToCheck.test(name)) {
+                return new MethodChecker(this.className, access, name, desc, wrongUsageCallback);
+            } else {
+                return super.visitMethod(access, name, desc, signature, exceptions);
+            }
+        }
+    }
+
+    private static class MethodChecker extends MethodVisitor {
+        private final String className;
+        private final Consumer<WrongLoggerUsage> wrongUsageCallback;
+        private boolean ignoreChecks;
+
+        public MethodChecker(String className, int access, String name, String desc, Consumer<WrongLoggerUsage> wrongUsageCallback) {
+            super(Opcodes.ASM5, new MethodNode(access, name, desc, null, null));
+            this.className = className;
+            this.wrongUsageCallback = wrongUsageCallback;
+        }
+
+        @Override
+        public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
+            if (IGNORE_CHECKS_ANNOTATION.equals(Type.getType(desc).getClassName())) {
+                ignoreChecks = true;
+            }
+            return super.visitAnnotation(desc, visible);
+        }
+
+        @Override
+        public void visitEnd() {
+            if (ignoreChecks == false) {
+                findBadLoggerUsages((MethodNode) mv);
+            }
+            super.visitEnd();
+        }
+
+        public void findBadLoggerUsages(MethodNode methodNode) {
+            Analyzer<BasicValue> stringPlaceHolderAnalyzer = new Analyzer<>(new PlaceHolderStringInterpreter());
+            Analyzer<BasicValue> arraySizeAnalyzer = new Analyzer<>(new ArraySizeInterpreter());
+            try {
+                stringPlaceHolderAnalyzer.analyze(className, methodNode);
+                arraySizeAnalyzer.analyze(className, methodNode);
+            } catch (AnalyzerException e) {
+                throw new RuntimeException("Internal error: failed in analysis step", e);
+            }
+            Frame<BasicValue>[] stringFrames = stringPlaceHolderAnalyzer.getFrames();
+            Frame<BasicValue>[] arraySizeFrames = arraySizeAnalyzer.getFrames();
+            AbstractInsnNode[] insns = methodNode.instructions.toArray();
+            int lineNumber = -1;
+            for (int i = 0; i < insns.length; i++) {
+                AbstractInsnNode insn = insns[i];
+                if (insn instanceof LineNumberNode) {
+                    LineNumberNode lineNumberNode = (LineNumberNode) insn;
+                    lineNumber = lineNumberNode.line;
+                }
+                if (insn.getOpcode() == Opcodes.INVOKEVIRTUAL) {
+                    MethodInsnNode methodInsn = (MethodInsnNode) insn;
+                    if (Type.getObjectType(methodInsn.owner).getClassName().equals(LOGGER_CLASS) == false) {
+                        continue;
+                    }
+                    if (LOGGER_METHODS.contains(methodInsn.name) == false) {
+                        continue;
+                    }
+                    Type[] argumentTypes = Type.getArgumentTypes(methodInsn.desc);
+                    BasicValue logMessageLengthObject = getStackValue(stringFrames[i], argumentTypes.length - 1); // first argument
+                    if (logMessageLengthObject instanceof PlaceHolderStringBasicValue == false) {
+                        wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
+                            "First argument must be a string constant so that we can statically ensure proper place holder usage"));
+                        continue;
+                    }
+                    PlaceHolderStringBasicValue logMessageLength = (PlaceHolderStringBasicValue) logMessageLengthObject;
+                    if (logMessageLength.minValue != logMessageLength.maxValue) {
+                        wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
+                            "Multiple log messages with conflicting number of place holders"));
+                        continue;
+                    }
+                    BasicValue varArgsSizeObject = getStackValue(arraySizeFrames[i], 0); // last argument
+                    if (varArgsSizeObject instanceof ArraySizeBasicValue == false) {
+                        wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
+                            "Could not determine size of varargs array"));
+                        continue;
+                    }
+                    ArraySizeBasicValue varArgsSize = (ArraySizeBasicValue) varArgsSizeObject;
+                    if (varArgsSize.minValue != varArgsSize.maxValue) {
+                        wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
+                            "Multiple parameter arrays with conflicting sizes"));
+                        continue;
+                    }
+                    assert logMessageLength.minValue == logMessageLength.maxValue && varArgsSize.minValue == varArgsSize.maxValue;
+                    if (logMessageLength.minValue != varArgsSize.minValue) {
+                        wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
+                            "Expected " + logMessageLength.minValue + " arguments but got " + varArgsSize.minValue));
+                        continue;
+                    }
+                }
+            }
+        }
+    }
+
+    private static int calculateNumberOfPlaceHolders(String message) {
+        int count = 0;
+        for (int i = 1; i < message.length(); i++) {
+            if (message.charAt(i - 1) == '{' && message.charAt(i) == '}') {
+                count++;
+                i += 1;
+            }
+        }
+        return count;
+    }
+
+    private static BasicValue getStackValue(Frame<BasicValue> f, int index) {
+        int top = f.getStackSize() - 1;
+        return index <= top ? f.getStack(top - index) : null;
+    }
+
+    private static class IntMinMaxTrackingBasicValue extends BasicValue {
+        protected final int minValue;
+        protected final int maxValue;
+
+        public IntMinMaxTrackingBasicValue(Type type, int value) {
+            super(type);
+            this.minValue = value;
+            this.maxValue = value;
+        }
+
+        public IntMinMaxTrackingBasicValue(Type type, int minValue, int maxValue) {
+            super(type);
+            this.minValue = minValue;
+            this.maxValue = maxValue;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            if (!super.equals(o)) return false;
+
+            IntMinMaxTrackingBasicValue that = (IntMinMaxTrackingBasicValue) o;
+
+            if (minValue != that.minValue) return false;
+            return maxValue == that.maxValue;
+
+        }
+
+        @Override
+        public int hashCode() {
+            int result = super.hashCode();
+            result = 31 * result + minValue;
+            result = 31 * result + maxValue;
+            return result;
+        }
+
+        @Override
+        public String toString() {
+            return "IntMinMaxTrackingBasicValue{" +
+                "minValue=" + minValue +
+                ", maxValue=" + maxValue +
+                '}';
+        }
+    }
+
+    private static final class PlaceHolderStringBasicValue extends IntMinMaxTrackingBasicValue {
+        public static final Type STRING_OBJECT_TYPE = Type.getObjectType("java/lang/String");
+
+        public PlaceHolderStringBasicValue(int placeHolders) {
+            super(STRING_OBJECT_TYPE, placeHolders);
+        }
+
+        public PlaceHolderStringBasicValue(int minPlaceHolders, int maxPlaceHolders) {
+            super(STRING_OBJECT_TYPE, minPlaceHolders, maxPlaceHolders);
+        }
+    }
+
+    private static final class ArraySizeBasicValue extends IntMinMaxTrackingBasicValue {
+        public ArraySizeBasicValue(Type type, int minArraySize, int maxArraySize) {
+            super(type, minArraySize, maxArraySize);
+        }
+    }
+
+    private static final class IntegerConstantBasicValue extends IntMinMaxTrackingBasicValue {
+        public IntegerConstantBasicValue(Type type, int constant) {
+            super(type, constant);
+        }
+
+        public IntegerConstantBasicValue(Type type, int minConstant, int maxConstant) {
+            super(type, minConstant, maxConstant);
+        }
+    }
+
+    private static final class PlaceHolderStringInterpreter extends BasicInterpreter {
+        @Override
+        public BasicValue newOperation(AbstractInsnNode insnNode) throws AnalyzerException {
+            if (insnNode.getOpcode() == Opcodes.LDC) {
+                Object constant = ((LdcInsnNode) insnNode).cst;
+                if (constant instanceof String) {
+                    return new PlaceHolderStringBasicValue(calculateNumberOfPlaceHolders((String) constant));
+                }
+            }
+            return super.newOperation(insnNode);
+        }
+
+        @Override
+        public BasicValue merge(BasicValue value1, BasicValue value2) {
+            if (value1 instanceof PlaceHolderStringBasicValue && value2 instanceof PlaceHolderStringBasicValue
+                && value1.equals(value2) == false) {
+                PlaceHolderStringBasicValue c1 = (PlaceHolderStringBasicValue) value1;
+                PlaceHolderStringBasicValue c2 = (PlaceHolderStringBasicValue) value2;
+                return new PlaceHolderStringBasicValue(Math.min(c1.minValue, c2.minValue), Math.max(c1.maxValue, c2.maxValue));
+            }
+            return super.merge(value1, value2);
+        }
+    }
+
+    private static final class ArraySizeInterpreter extends BasicInterpreter {
+        @Override
+        public BasicValue newOperation(AbstractInsnNode insnNode) throws AnalyzerException {
+            switch (insnNode.getOpcode()) {
+                case ICONST_0: return new IntegerConstantBasicValue(Type.INT_TYPE, 0);
+                case ICONST_1: return new IntegerConstantBasicValue(Type.INT_TYPE, 1);
+                case ICONST_2: return new IntegerConstantBasicValue(Type.INT_TYPE, 2);
+                case ICONST_3: return new IntegerConstantBasicValue(Type.INT_TYPE, 3);
+                case ICONST_4: return new IntegerConstantBasicValue(Type.INT_TYPE, 4);
+                case ICONST_5: return new IntegerConstantBasicValue(Type.INT_TYPE, 5);
+                case BIPUSH:
+                case SIPUSH: return new IntegerConstantBasicValue(Type.INT_TYPE, ((IntInsnNode)insnNode).operand);
+                case Opcodes.LDC: {
+                    Object constant = ((LdcInsnNode)insnNode).cst;
+                    if (constant instanceof Integer) {
+                        return new IntegerConstantBasicValue(Type.INT_TYPE, (Integer)constant);
+                    } else {
+                        return super.newOperation(insnNode);
+                    }
+                }
+                default: return super.newOperation(insnNode);
+            }
+        }
+
+        @Override
+        public BasicValue merge(BasicValue value1, BasicValue value2) {
+            if (value1 instanceof IntegerConstantBasicValue && value2 instanceof IntegerConstantBasicValue) {
+                IntegerConstantBasicValue c1 = (IntegerConstantBasicValue) value1;
+                IntegerConstantBasicValue c2 = (IntegerConstantBasicValue) value2;
+                return new IntegerConstantBasicValue(Type.INT_TYPE, Math.min(c1.minValue, c2.minValue), Math.max(c1.maxValue, c2.maxValue));
+            } else if (value1 instanceof ArraySizeBasicValue && value2 instanceof ArraySizeBasicValue) {
+                ArraySizeBasicValue c1 = (ArraySizeBasicValue) value1;
+                ArraySizeBasicValue c2 = (ArraySizeBasicValue) value2;
+                return new ArraySizeBasicValue(Type.INT_TYPE, Math.min(c1.minValue, c2.minValue), Math.max(c1.maxValue, c2.maxValue));
+            }
+            return super.merge(value1, value2);
+        }
+
+        @Override
+        public BasicValue unaryOperation(AbstractInsnNode insnNode, BasicValue value) throws AnalyzerException {
+            if (insnNode.getOpcode() == Opcodes.ANEWARRAY && value instanceof IntegerConstantBasicValue) {
+                IntegerConstantBasicValue constantBasicValue = (IntegerConstantBasicValue) value;
+                String desc = ((TypeInsnNode) insnNode).desc;
+                return new ArraySizeBasicValue(Type.getType("[" + Type.getObjectType(desc)), constantBasicValue.minValue,
+                    constantBasicValue.maxValue);
+            }
+            return super.unaryOperation(insnNode, value);
+        }
+
+        @Override
+        public BasicValue ternaryOperation(AbstractInsnNode insnNode, BasicValue value1, BasicValue value2, BasicValue value3)
+            throws AnalyzerException {
+            if (insnNode.getOpcode() == Opcodes.AASTORE && value1 instanceof ArraySizeBasicValue) {
+                return value1;
+            }
+            return super.ternaryOperation(insnNode, value1, value2, value3);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/percolator/PercolatorModule.java b/test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/SuppressForbidden.java
similarity index 58%
rename from core/src/main/java/org/elasticsearch/percolator/PercolatorModule.java
rename to test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/SuppressForbidden.java
index 68b8db55e31..995269e9f02 100644
--- a/core/src/main/java/org/elasticsearch/percolator/PercolatorModule.java
+++ b/test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/SuppressForbidden.java
@@ -16,18 +16,20 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.percolator;
 
-import org.elasticsearch.common.inject.AbstractModule;
+package org.elasticsearch.test.loggerusage;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
 
 /**
- *
+ * Annotation to suppress forbidden-apis errors inside a whole class, a method, or a field.
+ * Duplicated from core as main sources of logger-usage project have no dependencies on core
  */
-public class PercolatorModule extends AbstractModule {
-
-    @Override
-    protected void configure() {
-        bind(PercolateDocumentParser.class).asEagerSingleton();
-        bind(PercolatorService.class).asEagerSingleton();
-    }
+@Retention(RetentionPolicy.CLASS)
+@Target({ ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE })
+public @interface SuppressForbidden {
+    String reason();
 }
diff --git a/test/logger-usage/src/test/java/org/elasticsearch/test/loggerusage/ESLoggerUsageTests.java b/test/logger-usage/src/test/java/org/elasticsearch/test/loggerusage/ESLoggerUsageTests.java
new file mode 100644
index 00000000000..ab07ecbf45e
--- /dev/null
+++ b/test/logger-usage/src/test/java/org/elasticsearch/test/loggerusage/ESLoggerUsageTests.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.loggerusage;
+
+import org.elasticsearch.common.SuppressLoggerChecks;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.test.loggerusage.ESLoggerUsageChecker.WrongLoggerUsage;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Predicate;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class ESLoggerUsageTests extends ESTestCase {
+
+    public void testLoggerUsageChecks() throws IOException {
+        for (Method method : getClass().getMethods()) {
+            if (method.getDeclaringClass().equals(getClass())) {
+                if (method.getName().startsWith("check")) {
+                    logger.info("Checking logger usage for method {}", method.getName());
+                    InputStream classInputStream = getClass().getResourceAsStream(getClass().getSimpleName() + ".class");
+                    List<WrongLoggerUsage> errors = new ArrayList<>();
+                    ESLoggerUsageChecker.check(errors::add, classInputStream, Predicate.isEqual(method.getName()));
+                    if (method.getName().startsWith("checkFail")) {
+                        assertFalse("Expected " + method.getName() + " to have wrong ESLogger usage", errors.isEmpty());
+                    } else {
+                        assertTrue("Method " + method.getName() + " has unexpected ESLogger usage errors: " + errors, errors.isEmpty());
+                    }
+                } else {
+                    assertTrue("only allow methods starting with test or check in this class", method.getName().startsWith("test"));
+                }
+            }
+        }
+    }
+
+    public void testLoggerUsageCheckerCompatibilityWithESLogger() throws NoSuchMethodException {
+        assertThat(ESLoggerUsageChecker.LOGGER_CLASS, equalTo(ESLogger.class.getName()));
+        assertThat(ESLoggerUsageChecker.THROWABLE_CLASS, equalTo(Throwable.class.getName()));
+        int varargsMethodCount = 0;
+        for (Method method : ESLogger.class.getMethods()) {
+            if (method.isVarArgs()) {
+                // check that logger usage checks all varargs methods
+                assertThat(ESLoggerUsageChecker.LOGGER_METHODS, hasItem(method.getName()));
+                varargsMethodCount++;
+            }
+        }
+        // currently we have two overloaded methods for each of debug, info, ...
+        // if that changes, we might want to have another look at the usage checker
+        assertThat(varargsMethodCount, equalTo(ESLoggerUsageChecker.LOGGER_METHODS.size() * 2));
+
+        // check that signature is same as we expect in the usage checker
+        for (String methodName : ESLoggerUsageChecker.LOGGER_METHODS) {
+            assertThat(ESLogger.class.getMethod(methodName, String.class, Object[].class), notNullValue());
+            assertThat(ESLogger.class.getMethod(methodName, String.class, Throwable.class, Object[].class), notNullValue());
+        }
+    }
+
+    public void checkNumberOfArguments1() {
+        logger.info("Hello {}", "world");
+    }
+
+    public void checkFailNumberOfArguments1() {
+        logger.info("Hello {}");
+    }
+
+    @SuppressLoggerChecks(reason = "test ignore functionality")
+    public void checkIgnoreWhenAnnotationPresent() {
+        logger.info("Hello {}");
+    }
+
+    public void checkNumberOfArguments2() {
+        logger.info("Hello {}, {}, {}", "world", 2, "third argument");
+    }
+
+    public void checkFailNumberOfArguments2() {
+        logger.info("Hello {}, {}", "world", 2, "third argument");
+    }
+
+    public void checkNumberOfArguments3() {
+        // long argument list (> 5), emits different bytecode
+        logger.info("Hello {}, {}, {}, {}, {}, {}, {}", "world", 2, "third argument", 4, 5, 6, new String("last arg"));
+    }
+
+    public void checkFailNumberOfArguments3() {
+        logger.info("Hello {}, {}, {}, {}, {}, {}, {}", "world", 2, "third argument", 4, 5, 6, 7, new String("last arg"));
+    }
+
+    public void checkOrderOfExceptionArgument() {
+        logger.info("Hello", new Exception());
+    }
+
+    public void checkOrderOfExceptionArgument1() {
+        logger.info("Hello {}", new Exception(), "world");
+    }
+
+    public void checkFailOrderOfExceptionArgument1() {
+        logger.info("Hello {}", "world", new Exception());
+    }
+
+    public void checkOrderOfExceptionArgument2() {
+        logger.info("Hello {}, {}", new Exception(), "world", 42);
+    }
+
+    public void checkFailOrderOfExceptionArgument2() {
+        logger.info("Hello {}, {}", "world", 42, new Exception());
+    }
+
+    public void checkFailNonConstantMessage(boolean b) {
+        logger.info(Boolean.toString(b));
+    }
+
+    public void checkComplexUsage(boolean b) {
+        String message = "Hello {}, {}";
+        Object[] args = new Object[] { "world", 42 };
+        if (b) {
+            message = "also two args {}{}";
+            args = new Object[] { "world", 43 };
+        }
+        logger.info(message, args);
+    }
+
+    public void checkFailComplexUsage1(boolean b) {
+        String message = "Hello {}, {}";
+        Object[] args = new Object[] { "world", 42 };
+        if (b) {
+            message = "just one arg {}";
+            args = new Object[] { "world", 43 };
+        }
+        logger.info(message, args);
+    }
+
+    public void checkFailComplexUsage2(boolean b) {
+        String message = "Hello {}, {}";
+        Object[] args = new Object[] { "world", 42 };
+        if (b) {
+            message = "also two args {}{}";
+            args = new Object[] { "world", 43, "another argument" };
+        }
+        logger.info(message, args);
+    }
+}